From c86e00bbea8681fe7e7fbe645ef67c594c5c335e Mon Sep 17 00:00:00 2001 From: William Morriss Date: Wed, 29 Oct 2025 16:26:45 -0500 Subject: [PATCH 01/28] Squashed branch 'create-contexts' (https://github.com/FilOzone/synapse-sdk/pull/325) onto master --- packages/synapse-sdk/src/storage/context.ts | 85 ++++++- packages/synapse-sdk/src/storage/manager.ts | 10 + .../test/mocks/jsonrpc/service-registry.ts | 112 ++++++++- .../src/test/sp-registry-service.test.ts | 1 - packages/synapse-sdk/src/test/storage.test.ts | 3 +- packages/synapse-sdk/src/test/synapse.test.ts | 219 +++++++++++++++++- packages/synapse-sdk/src/test/test-utils.ts | 12 +- packages/synapse-sdk/src/types.ts | 38 ++- 8 files changed, 459 insertions(+), 21 deletions(-) diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index 5e3baf62b..6bbacb3ee 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -33,6 +33,7 @@ import { SPRegistryService } from '../sp-registry/index.ts' import type { ProviderInfo } from '../sp-registry/types.ts' import type { Synapse } from '../synapse.ts' import type { + CreateContextsOptions, DownloadOptions, EnhancedDataSetInfo, MetadataEntry, @@ -175,6 +176,78 @@ export class StorageContext { this._pdpServer = new PDPServer(authHelper, provider.products.PDP.data.serviceURL) } + static async createContexts( + synapse: Synapse, + warmStorageService: WarmStorageService, + options: CreateContextsOptions = {} + ): Promise { + const count = options?.count ?? 2 + const resolutions: ProviderSelectionResult[] = [] + const clientAddress = await synapse.getClient().getAddress() + const registryAddress = warmStorageService.getServiceProviderRegistryAddress() + const spRegistry = new SPRegistryService(synapse.getProvider(), registryAddress) + const providerResolver = new ProviderResolver(warmStorageService, spRegistry) + if (options?.dataSetIds) { + for (const dataSetId of options.dataSetIds) { + const resolution = await StorageContext.resolveByDataSetId( + dataSetId, + warmStorageService, + providerResolver, + clientAddress, + { + withCDN: options.withCDN, + withIpni: options.withIpni, + dev: options.dev, + metadata: options.metadata, + } + ) + resolutions.push(resolution) + if (resolutions.length >= count) { + break + } + } + } + if (resolutions.length < count) { + if (options?.providerIds) { + for (const providerId of options.providerIds) { + const resolution = await StorageContext.resolveByProviderId( + clientAddress, + providerId, + options.metadata ?? {}, + warmStorageService, + providerResolver, + options.forceCreateDataSets + ) + resolutions.push(resolution) + if (resolutions.length >= count) { + break + } + } + } else if (options?.providerAddresses) { + for (const providerAddress of options.providerAddresses) { + const resolution = await StorageContext.resolveByProviderAddress( + providerAddress, + warmStorageService, + providerResolver, + clientAddress, + options.metadata ?? {}, + options.forceCreateDataSets + ) + resolutions.push(resolution) + if (resolutions.length >= count) { + break + } + } + } + } + return await Promise.all( + resolutions.map( + async (resolution) => + await StorageContext.createWithSelectedProvider(resolution, synapse, warmStorageService, options) + ) + ) + } + /** * Static factory method to create a StorageContext * Handles provider selection and data set selection/creation @@ -197,6 +270,15 @@ export class StorageContext { options ) + return await StorageContext.createWithSelectedProvider(resolution, synapse, warmStorageService, options) + } + + private static async createWithSelectedProvider( + resolution: ProviderSelectionResult, + synapse: Synapse, + warmStorageService: WarmStorageService, + options: StorageServiceOptions = {} + ): Promise { // Notify callback about provider selection try { options.callbacks?.onProviderSelected?.(resolution.provider) @@ -233,8 +315,7 @@ export class StorageContext { providerResolver: ProviderResolver, options: StorageServiceOptions ): Promise { - const client = synapse.getClient() - const clientAddress = await client.getAddress() + const clientAddress = await synapse.getClient().getAddress() // Handle explicit data set ID selection (highest priority) if (options.dataSetId != null && options.forceCreateDataSet !== true) { diff --git a/packages/synapse-sdk/src/storage/manager.ts b/packages/synapse-sdk/src/storage/manager.ts index 2b2dcc65a..5ee12911e 100644 --- a/packages/synapse-sdk/src/storage/manager.ts +++ b/packages/synapse-sdk/src/storage/manager.ts @@ -25,6 +25,7 @@ import { asPieceCID, downloadAndValidate } from '../piece/index.ts' import { SPRegistryService } from '../sp-registry/index.ts' import type { Synapse } from '../synapse.ts' import type { + CreateContextsOptions, DownloadOptions, EnhancedDataSetInfo, PieceCID, @@ -231,6 +232,15 @@ export class StorageManager { return await StorageContext.performPreflightCheck(this._warmStorageService, this._synapse.payments, size, withCDN) } + async createContexts(options?: CreateContextsOptions): Promise { + return await StorageContext.createContexts(this._synapse, this._warmStorageService, { + ...options, + withCDN: options?.withCDN ?? this._withCDN, + withIpni: options?.withIpni ?? this._withIpni, + dev: options?.dev ?? this._dev, + }) + } + /** * Create a new storage context with specified options */ diff --git a/packages/synapse-sdk/src/test/mocks/jsonrpc/service-registry.ts b/packages/synapse-sdk/src/test/mocks/jsonrpc/service-registry.ts index fd52ee935..038312ea1 100644 --- a/packages/synapse-sdk/src/test/mocks/jsonrpc/service-registry.ts +++ b/packages/synapse-sdk/src/test/mocks/jsonrpc/service-registry.ts @@ -1,7 +1,12 @@ /** biome-ignore-all lint/style/noNonNullAssertion: testing */ +import type { PDPOffering } from '@filoz/synapse-core/utils' +import { encodePDPCapabilities } from '@filoz/synapse-core/utils' import type { ExtractAbiFunction } from 'abitype' -import { decodeFunctionData, encodeAbiParameters, type Hex } from 'viem' +import { assert } from 'chai' +import type { Hex } from 'viem' +import { decodeFunctionData, encodeAbiParameters } from 'viem' +import type { PDPServiceInfo } from '../../../sp-registry/types.ts' import { CONTRACT_ABIS } from '../../../utils/constants.ts' import type { AbiToType, JSONRPCOptions } from './types.ts' @@ -33,6 +38,111 @@ export interface ServiceRegistryOptions { ) => AbiToType } +export type ServiceProviderInfoView = AbiToType[0] +export type ProviderWithProduct = AbiToType[0] + +const EMPTY_PROVIDER_INFO = { + serviceProvider: '0x0000000000000000000000000000000000000000', + payee: '0x0000000000000000000000000000000000000000', + name: '', + description: '', + isActive: false, +} as const + +const EMPTY_PROVIDER_INFO_VIEW: ServiceProviderInfoView = { + providerId: 0n, + info: EMPTY_PROVIDER_INFO, +} + +const EMPTY_PROVIDER_WITH_PRODUCT: [ProviderWithProduct] = [ + { + providerId: 0n, + providerInfo: EMPTY_PROVIDER_INFO, + product: { + productType: 0, + capabilityKeys: [], + isActive: false, + }, + productCapabilityValues: [] as Hex[], + }, +] + +export function mockServiceProviderRegistry( + providers: ServiceProviderInfoView[], + services?: (PDPServiceInfo | null)[] +): ServiceRegistryOptions { + assert.isAtMost(services?.length ?? 0, providers.length) + return { + getProvider: ([providerId]) => { + if (providerId < 0n || providerId > providers.length) { + throw new Error('Provider does not exist') + } + for (const provider of providers) { + if (providerId === provider.providerId) { + return [provider] + } + } + throw new Error('Provider not found') + }, + getProviderWithProduct: ([providerId, productType]) => { + if (!services) { + return EMPTY_PROVIDER_WITH_PRODUCT + } + for (let i = 0; i < services.length; i++) { + if (providers[i].providerId === providerId) { + const providerInfo = providers[i].info + const service = services[i] + if (service == null) { + return [ + { + providerId, + providerInfo, + product: { + productType: 0, + capabilityKeys: [], + isActive: false, + }, + productCapabilityValues: [] as Hex[], + }, + ] + } + const [capabilityKeys, productCapabilityValues] = encodePDPCapabilities(service.offering) + return [ + { + providerId, + providerInfo, + product: { + productType: 0, // PDP + capabilityKeys, + isActive: true, + }, + productCapabilityValues, + }, + ] + } + } + return EMPTY_PROVIDER_WITH_PRODUCT + }, + // TODO getProvidersByProductType + getProviderByAddress: ([address]) => { + for (const provider of providers) { + if (address === provider.info.serviceProvider) { + return [provider] + } + } + return [EMPTY_PROVIDER_INFO_VIEW] + }, + getProviderIdByAddress: ([address]) => { + for (const provider of providers) { + if (address === provider.info.serviceProvider) { + return [provider.providerId] + } + } + return [0n] + }, + } +} + /** * Handle service provider registry calls */ diff --git a/packages/synapse-sdk/src/test/sp-registry-service.test.ts b/packages/synapse-sdk/src/test/sp-registry-service.test.ts index da82e6e7c..c44b2428a 100644 --- a/packages/synapse-sdk/src/test/sp-registry-service.test.ts +++ b/packages/synapse-sdk/src/test/sp-registry-service.test.ts @@ -55,7 +55,6 @@ describe('SPRegistryService', () => { return { providerId: 1, info: { - id: BigInt(1), serviceProvider: mockProviderAddress, payee: mockProviderAddress, name: 'Test Provider', diff --git a/packages/synapse-sdk/src/test/storage.test.ts b/packages/synapse-sdk/src/test/storage.test.ts index 1ed4b43ac..9aeb6a5ec 100644 --- a/packages/synapse-sdk/src/test/storage.test.ts +++ b/packages/synapse-sdk/src/test/storage.test.ts @@ -5,6 +5,7 @@ import { StorageContext } from '../storage/context.ts' import type { Synapse } from '../synapse.ts' import type { PieceCID, ProviderInfo } from '../types.ts' import { SIZE_CONSTANTS } from '../utils/constants.ts' +import { ADDRESSES } from './mocks/jsonrpc/index.ts' import { createMockProviderInfo, createSimpleProvider, setupProviderRegistryMocks } from './test-utils.ts' // Create a mock Ethereum provider that doesn't try to connect @@ -143,7 +144,7 @@ function createMockWarmStorageService(providers: ProviderInfo[], dataSets: any[] return provider?.id ?? 0 }, getApprovedProvider: async (id: number) => providers.find((p) => p.id === id) ?? null, - getServiceProviderRegistryAddress: () => '0x0000000000000000000000000000000000000001', + getServiceProviderRegistryAddress: () => ADDRESSES.calibration.spRegistry, getApprovedProviderIds: async () => providers.map((p) => p.id), isProviderIdApproved: async (id: number) => providers.some((p) => p.id === id), getDataSetMetadata: async (dataSetId: number) => { diff --git a/packages/synapse-sdk/src/test/synapse.test.ts b/packages/synapse-sdk/src/test/synapse.test.ts index 8887250c2..b0e762bbf 100644 --- a/packages/synapse-sdk/src/test/synapse.test.ts +++ b/packages/synapse-sdk/src/test/synapse.test.ts @@ -4,6 +4,8 @@ * Basic tests for Synapse class */ +import type { PDPOffering } from '@filoz/synapse-core/utils' +import { capabilitiesListToObject, encodePDPCapabilities } from '@filoz/synapse-core/utils' import { assert } from 'chai' import { ethers } from 'ethers' import { setup } from 'iso-web/msw' @@ -12,9 +14,12 @@ import pDefer from 'p-defer' import { type Address, bytesToHex, type Hex, isAddressEqual, numberToBytes, parseUnits, stringToHex } from 'viem' import { PaymentsService } from '../payments/index.ts' import { PDP_PERMISSIONS } from '../session/key.ts' +import type { PDPServiceInfo } from '../sp-registry/types.ts' import { Synapse } from '../synapse.ts' import { makeDataSetCreatedLog } from './mocks/events.ts' import { ADDRESSES, JSONRPC, PRIVATE_KEYS, presets } from './mocks/jsonrpc/index.ts' +import type { ServiceProviderInfoView } from './mocks/jsonrpc/service-registry.ts' +import { mockServiceProviderRegistry } from './mocks/jsonrpc/service-registry.ts' import { createDataSetHandler, dataSetCreationStatusHandler, type PDPMockOptions } from './mocks/pdp/handlers.ts' import { PING } from './mocks/ping.ts' @@ -327,10 +332,7 @@ describe('Synapse', () => { }, payments: { ...presets.basic.payments, - operatorApprovals: (args) => { - const token = args[0] - const client = args[1] - const operator = args[2] + operatorApprovals: ([token, client, operator]) => { assert.equal(token, ADDRESSES.calibration.usdfcToken) assert.equal(client, signerAddress) assert.equal(operator, ADDRESSES.calibration.warmStorage) @@ -343,9 +345,7 @@ describe('Synapse', () => { BigInt(28800), // maxLockupPeriod ] }, - accounts: (args) => { - const token = args[0] - const user = args[1] + accounts: ([token, user]) => { assert.equal(user, signerAddress) assert.equal(token, ADDRESSES.calibration.usdfcToken) return [BigInt(127001 * 635000000), BigInt(0), BigInt(0), BigInt(0)] @@ -797,4 +797,209 @@ describe('Synapse', () => { } }) }) + + describe('createContexts', () => { + let synapse: Synapse + const mockProviders: ServiceProviderInfoView[] = [ + { + providerId: 1n, + info: { + serviceProvider: ADDRESSES.serviceProvider1, + payee: ADDRESSES.serviceProvider1, + name: 'serviceProvider1', + description: 'mockProviders[0]', + isActive: true, + }, + }, + { + providerId: 2n, + info: { + serviceProvider: ADDRESSES.serviceProvider2, + payee: ADDRESSES.serviceProvider2, + name: 'serviceProvider2', + description: 'mockProviders[1]', + isActive: true, + }, + }, + ] + const offering1: PDPOffering = { + serviceURL: 'http://serviceProvider1.com', + minPieceSizeInBytes: 0n, + maxPieceSizeInBytes: 1n << 32n, + ipniPiece: false, + ipniIpfs: false, + storagePricePerTibPerDay: 1000n, + minProvingPeriodInEpochs: 3n, + location: 'narnia', + paymentTokenAddress: ADDRESSES.calibration.usdfcToken, + } + const offering2: PDPOffering = { + serviceURL: 'http://serviceProvider2.org', + minPieceSizeInBytes: 0n, + maxPieceSizeInBytes: 1n << 32n, + ipniPiece: false, + ipniIpfs: false, + storagePricePerTibPerDay: 1000n, + minProvingPeriodInEpochs: 3n, + location: 'krypton', + paymentTokenAddress: ADDRESSES.calibration.usdfcToken, + } + + const mockPDPProducts: PDPServiceInfo[] = [ + { + offering: offering1, + capabilities: capabilitiesListToObject(...encodePDPCapabilities(offering1)), + isActive: true, + }, + { + offering: offering2, + capabilities: capabilitiesListToObject(...encodePDPCapabilities(offering2)), + isActive: true, + }, + ] + + const DATA_SET_ID = 7 + + const FAKE_TX_HASH = '0x3816d82cb7a6f5cde23f4d63c0763050d13c6b6dc659d0a7e6eba80b0ec76a18' + + const FAKE_TX = { + hash: FAKE_TX_HASH, + from: ADDRESSES.serviceProvider1, + gas: '0x5208', + value: '0x0', + nonce: '0x444', + input: '0x', + v: '0x01', + r: '0x4e2eef88cc6f2dc311aa3b1c8729b6485bd606960e6ae01522298278932c333a', + s: '0x5d0e08d8ecd6ed8034aa956ff593de9dc1d392e73909ef0c0f828918b58327c9', + } + + const FAKE_RECEIPT = { + ...FAKE_TX, + transactionHash: FAKE_TX_HASH, + transactionIndex: '0x10', + blockHash: '0xb91b7314248aaae06f080ad427dbae78b8c5daf72b2446cf843739aef80c6417', + status: '0x1', + blockNumber: '0x127001', + cumulativeGasUsed: '0x52080', + gasUsed: '0x5208', + logs: [makeDataSetCreatedLog(DATA_SET_ID, 1)], + } + + beforeEach(async () => { + server.use( + JSONRPC({ + ...presets.basic, + serviceRegistry: mockServiceProviderRegistry(mockProviders, mockPDPProducts), + eth_getTransactionByHash: (params) => { + const hash = params[0] + assert.equal(hash, FAKE_TX_HASH) + return FAKE_TX + }, + eth_getTransactionReceipt: (params) => { + const hash = params[0] + assert.equal(hash, FAKE_TX_HASH) + return FAKE_RECEIPT + }, + }) + ) + synapse = await Synapse.create({ signer }) + for (const { offering } of mockPDPProducts) { + const pdpOptions: PDPMockOptions = { + baseUrl: offering.serviceURL, + } + server.use(createDataSetHandler(FAKE_TX_HASH, pdpOptions)) + server.use( + dataSetCreationStatusHandler( + FAKE_TX_HASH, + { + ok: true, + dataSetId: DATA_SET_ID, + createMessageHash: '', + dataSetCreated: true, + service: '', + txStatus: '', + }, + pdpOptions + ) + ) + } + }) + + it('selects specified providerIds', async () => { + const contexts = await synapse.storage.createContexts({ + providerIds: [mockProviders[0].providerId, mockProviders[1].providerId].map(Number), + }) + assert.equal(contexts.length, 2) + assert.equal(BigInt(contexts[0].provider.id), mockProviders[0].providerId) + assert.equal(BigInt(contexts[1].provider.id), mockProviders[1].providerId) + // created new data sets; got the mocked data set id + assert.equal((contexts[0] as any)._dataSetId, DATA_SET_ID) + assert.equal((contexts[1] as any)._dataSetId, DATA_SET_ID) + }) + + it('fails when provided an invalid providerId', async () => { + try { + await synapse.storage.createContexts({ + providerIds: [3, 4], + }) + assert.fail('Expected createContexts to fail for invalid specified providerIds') + } catch (error: any) { + assert.include(error.message, 'Provider does not exist') + } + }) + + it('selects providers specified by address', async () => { + const contexts = await synapse.storage.createContexts({ + providerAddresses: [mockProviders[1].info.serviceProvider, mockProviders[0].info.serviceProvider], + }) + assert.equal(contexts.length, 2) + assert.equal(BigInt(contexts[1].provider.id), mockProviders[0].providerId) + assert.equal(BigInt(contexts[0].provider.id), mockProviders[1].providerId) + // created new data sets; got the mocked data set id + assert.equal((contexts[1] as any)._dataSetId, DATA_SET_ID) + assert.equal((contexts[0] as any)._dataSetId, DATA_SET_ID) + }) + + it('fails when provided an invalid provider address', async () => { + try { + await synapse.storage.createContexts({ + providerAddresses: [ADDRESSES.client1], + }) + assert.fail('Expected createContexts to fail for invalid specified provider address') + } catch (error: any) { + assert.equal( + error?.message, + `StorageContext resolveByProviderAddress failed: Provider ${ADDRESSES.client1} is not currently approved` + ) + } + }) + + it('selects providers specified by data set id', async () => { + const contexts = await synapse.storage.createContexts({ + count: 1, + dataSetIds: [1], + }) + assert.equal(contexts.length, 1) + assert.equal(contexts[0].provider.id, 1) + assert.equal((contexts[0] as any)._dataSetId, 1n) + }) + + it('fails when provided an invalid data set id', async () => { + for (const dataSetId of [0, 2]) { + try { + await synapse.storage.createContexts({ + count: 1, + dataSetIds: [dataSetId], + }) + assert.fail('Expected createContexts to fail for invalid specified data set id') + } catch (error: any) { + assert.equal( + error?.message, + `StorageContext resolveByDataSetId failed: Data set ${dataSetId} not found, not owned by ${ADDRESSES.client1}, or not managed by the current WarmStorage contract` + ) + } + } + }) + }) }) diff --git a/packages/synapse-sdk/src/test/test-utils.ts b/packages/synapse-sdk/src/test/test-utils.ts index fb4420786..a3a13e2d2 100644 --- a/packages/synapse-sdk/src/test/test-utils.ts +++ b/packages/synapse-sdk/src/test/test-utils.ts @@ -21,6 +21,7 @@ import type { ProviderInfo } from '../sp-registry/types.ts' import { CONTRACT_ABIS, CONTRACT_ADDRESSES, SIZE_CONSTANTS, TIME_CONSTANTS } from '../utils/constants.ts' import { ProviderResolver } from '../utils/provider-resolver.ts' import type { WarmStorageService } from '../warm-storage/index.ts' +import { ADDRESSES } from './mocks/jsonrpc/index.ts' /** * Addresses used by testing @@ -152,7 +153,7 @@ export function createMockProvider(chainId: number = 314159): ethers.Provider { } // serviceProviderRegistry() - function selector: 0x05f892ec if (data?.startsWith('0x05f892ec') === true) { - return ethers.AbiCoder.defaultAbiCoder().encode(['address'], ['0x0000000000000000000000000000000000000001']) + return ethers.AbiCoder.defaultAbiCoder().encode(['address'], [ADDRESSES.calibration.spRegistry]) } // sessionKeyRegistry() - function selector: 0x9f6aa572 if (data?.startsWith('0x9f6aa572') === true) { @@ -460,7 +461,7 @@ export function createCustomMulticall3Mock( customAddresses?.usdfcToken ?? CONTRACT_ADDRESSES.USDFC.calibration, // usdfcToken customAddresses?.filCDN ?? '0x0000000000000000000000000000000000000000', // filCDN (not used) customAddresses?.viewContract ?? MOCK_ADDRESSES.WARM_STORAGE_VIEW, // viewContract - customAddresses?.spRegistry ?? '0x0000000000000000000000000000000000000001', // spRegistry + customAddresses?.spRegistry ?? ADDRESSES.calibration.spRegistry, // spRegistry customAddresses?.sessionKeyRegistry ?? MOCK_ADDRESSES.SESSION_KEY_REGISTRY, // sessionKeyRegistry ] @@ -640,10 +641,7 @@ export function setupProviderRegistryMocks( if (callData.startsWith('0xab2b3ae5')) { return { success: true, - returnData: ethers.AbiCoder.defaultAbiCoder().encode( - ['address'], - ['0x0000000000000000000000000000000000000001'] - ), + returnData: ethers.AbiCoder.defaultAbiCoder().encode(['address'], [ADDRESSES.calibration.spRegistry]), } } } @@ -661,7 +659,7 @@ export function setupProviderRegistryMocks( // Mock getProvider(uint256) calls to SPRegistry // Check if it's to the SPRegistry address - if (callData.startsWith('0x5c42d079') && target === '0x0000000000000000000000000000000000000001') { + if (callData.startsWith('0x5c42d079') && target === ADDRESSES.calibration.spRegistry) { const providerId = parseInt(callData.slice(10, 74), 16) const provider = providers.find((p) => p.id === providerId) if (provider) { diff --git a/packages/synapse-sdk/src/types.ts b/packages/synapse-sdk/src/types.ts index 82c435512..c6716ac15 100644 --- a/packages/synapse-sdk/src/types.ts +++ b/packages/synapse-sdk/src/types.ts @@ -298,6 +298,42 @@ export interface StorageContextCallbacks { onDataSetResolved?: (info: { isExisting: boolean; dataSetId: number; provider: ProviderInfo }) => void } +export interface CreateContextsOptions { + /** Number of contexts to create (optional, defaults to 2) */ + count?: number + /** + * Specific provider IDs to use (if not using providerAddresses) + * Must be no longer than count + */ + providerIds?: number[] + /** + * Specific provider addresses to use (if not using providerIds) + * Must be no longer than count + */ + providerAddresses?: string[] + /** + * Specific data set IDs to use + * Cannot be used with provider options + * Must be no longer than count + */ + dataSetIds?: number[] + /** Whether to enable CDN services */ + withCDN?: boolean + withIpni?: boolean + dev?: boolean + /** + * Custom metadata for the data sets (key-value pairs) + * When smart-selecting data sets, this metadata will be used to match. + */ + metadata?: Record + /** Create new data sets, even if candidates exist */ + forceCreateDataSets?: boolean + /** Callbacks for creation process (will need to change to handle multiples) */ + callbacks?: StorageContextCallbacks + /** Maximum number of uploads to process in a single batch (default: 32, minimum: 1) */ + uploadBatchSize?: number +} + /** * Options for creating or selecting a storage context * @@ -531,8 +567,6 @@ export interface ProviderSelectionResult { provider: ProviderInfo /** Selected data set ID */ dataSetId: number - /** Whether this is a new data set that was created */ - isNewDataSet?: boolean /** Whether this is an existing data set */ isExisting?: boolean /** Data set metadata */ From 1b75a253f47825136903582bd84eacdd4fb0297a Mon Sep 17 00:00:00 2001 From: William Morriss Date: Wed, 29 Oct 2025 22:52:35 -0500 Subject: [PATCH 02/28] mockServiceProviderRegistry: getProvidersByProductType --- .../test/mocks/jsonrpc/service-registry.ts | 59 +++++++++++++++++-- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/packages/synapse-sdk/src/test/mocks/jsonrpc/service-registry.ts b/packages/synapse-sdk/src/test/mocks/jsonrpc/service-registry.ts index 038312ea1..a314bac27 100644 --- a/packages/synapse-sdk/src/test/mocks/jsonrpc/service-registry.ts +++ b/packages/synapse-sdk/src/test/mocks/jsonrpc/service-registry.ts @@ -1,6 +1,5 @@ /** biome-ignore-all lint/style/noNonNullAssertion: testing */ -import type { PDPOffering } from '@filoz/synapse-core/utils' import { encodePDPCapabilities } from '@filoz/synapse-core/utils' import type { ExtractAbiFunction } from 'abitype' import { assert } from 'chai' @@ -27,6 +26,11 @@ export type getProviderWithProduct = ExtractAbiFunction< 'getProviderWithProduct' > +export type getProvidersByProductType = ExtractAbiFunction< + typeof CONTRACT_ABIS.SERVICE_PROVIDER_REGISTRY, + 'getProvidersByProductType' +> + export interface ServiceRegistryOptions { getProviderByAddress?: (args: AbiToType) => AbiToType getProviderIdByAddress?: ( @@ -36,6 +40,9 @@ export interface ServiceRegistryOptions { getProviderWithProduct?: ( args: AbiToType ) => AbiToType + getProvidersByProductType?: ( + args: AbiToType + ) => AbiToType } export type ServiceProviderInfoView = AbiToType[0] @@ -98,7 +105,7 @@ export function mockServiceProviderRegistry( providerId, providerInfo, product: { - productType: 0, + productType, capabilityKeys: [], isActive: false, }, @@ -112,7 +119,7 @@ export function mockServiceProviderRegistry( providerId, providerInfo, product: { - productType: 0, // PDP + productType, capabilityKeys, isActive: true, }, @@ -123,7 +130,51 @@ export function mockServiceProviderRegistry( } return EMPTY_PROVIDER_WITH_PRODUCT }, - // TODO getProvidersByProductType + getProvidersByProductType: ([productType, onlyActive, offset, limit]) => { + if (!services) { + return [ + { + providers: [] as ProviderWithProduct[], + hasMore: false, + }, + ] + } + const filteredProviders: ProviderWithProduct[] = [] + for (let i = 0; i < services.length; i++) { + const providerInfoView = providers[i] + const providerId = providerInfoView.providerId + const providerInfo = providers[i].info + if (onlyActive && !providerInfo.isActive) { + continue + } + const service = services[i] + if (service == null || !service.isActive) { + continue + } + if (productType !== 0) { + // this mock currently only supports PDP + continue + } + const [capabilityKeys, productCapabilityValues] = encodePDPCapabilities(service.offering) + filteredProviders.push({ + providerId, + providerInfo, + product: { + productType: 0, // PDP + capabilityKeys, + isActive: service.isActive, + }, + productCapabilityValues, + }) + } + const hasMore = offset + limit >= filteredProviders.length + return [ + { + providers: filteredProviders.slice(Number(offset), Number(offset + limit)), + hasMore, + }, + ] + }, getProviderByAddress: ([address]) => { for (const provider of providers) { if (address === provider.info.serviceProvider) { From 53e4374e5d1403adb7326fa000a7b171f3cb4273 Mon Sep 17 00:00:00 2001 From: William Morriss Date: Wed, 29 Oct 2025 23:29:41 -0500 Subject: [PATCH 03/28] update tests: create data set is now delayed until first add piece --- packages/synapse-sdk/src/test/synapse.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/synapse-sdk/src/test/synapse.test.ts b/packages/synapse-sdk/src/test/synapse.test.ts index b0e762bbf..73d274647 100644 --- a/packages/synapse-sdk/src/test/synapse.test.ts +++ b/packages/synapse-sdk/src/test/synapse.test.ts @@ -933,9 +933,9 @@ describe('Synapse', () => { assert.equal(contexts.length, 2) assert.equal(BigInt(contexts[0].provider.id), mockProviders[0].providerId) assert.equal(BigInt(contexts[1].provider.id), mockProviders[1].providerId) - // created new data sets; got the mocked data set id - assert.equal((contexts[0] as any)._dataSetId, DATA_SET_ID) - assert.equal((contexts[1] as any)._dataSetId, DATA_SET_ID) + // should create new data sets + assert.equal((contexts[0] as any)._dataSetId, undefined) + assert.equal((contexts[1] as any)._dataSetId, undefined) }) it('fails when provided an invalid providerId', async () => { @@ -956,9 +956,9 @@ describe('Synapse', () => { assert.equal(contexts.length, 2) assert.equal(BigInt(contexts[1].provider.id), mockProviders[0].providerId) assert.equal(BigInt(contexts[0].provider.id), mockProviders[1].providerId) - // created new data sets; got the mocked data set id - assert.equal((contexts[1] as any)._dataSetId, DATA_SET_ID) - assert.equal((contexts[0] as any)._dataSetId, DATA_SET_ID) + // should create new data sets + assert.equal((contexts[1] as any)._dataSetId, undefined) + assert.equal((contexts[0] as any)._dataSetId, undefined) }) it('fails when provided an invalid provider address', async () => { From 7411bd072c2f157aeff813867448892317521529 Mon Sep 17 00:00:00 2001 From: William Morriss Date: Thu, 30 Oct 2025 14:07:41 -0500 Subject: [PATCH 04/28] use smartSelectProvider for the remaining providers --- packages/synapse-sdk/src/storage/context.ts | 20 ++++++++ packages/synapse-sdk/src/test/synapse.test.ts | 51 +++++++++++++++++-- packages/synapse-sdk/src/types.ts | 2 + 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index 5d3d061dc..214b69205 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -239,6 +239,26 @@ export class StorageContext { } } } + if (resolutions.length <= count) { + const excludeProviderIds = [ + ...(options.excludeProviderIds ?? []), + ...resolutions.map((resolution) => resolution.provider.id), + ] + for (let i = resolutions.length; i < count; i++) { + const resolution = await StorageContext.smartSelectProvider( + clientAddress, + options.metadata ?? {}, + warmStorageService, + spRegistry, + excludeProviderIds, + options.forceCreateDataSets, + options.withIpni, + options.dev + ) + excludeProviderIds.push(resolution.provider.id) + resolutions.push(resolution) + } + } return await Promise.all( resolutions.map( async (resolution) => diff --git a/packages/synapse-sdk/src/test/synapse.test.ts b/packages/synapse-sdk/src/test/synapse.test.ts index d0f58e93b..e0ed9fe47 100644 --- a/packages/synapse-sdk/src/test/synapse.test.ts +++ b/packages/synapse-sdk/src/test/synapse.test.ts @@ -916,6 +916,7 @@ describe('Synapse', () => { const pdpOptions: PDPMockOptions = { baseUrl: offering.serviceURL, } + server.use(PING(pdpOptions)) server.use(createDataSetHandler(FAKE_TX_HASH, pdpOptions)) server.use( dataSetCreationStatusHandler( @@ -984,13 +985,13 @@ describe('Synapse', () => { }) it('selects providers specified by data set id', async () => { - const contexts = await synapse.storage.createContexts({ + const contexts1 = await synapse.storage.createContexts({ count: 1, dataSetIds: [1], }) - assert.equal(contexts.length, 1) - assert.equal(contexts[0].provider.id, 1) - assert.equal((contexts[0] as any)._dataSetId, 1n) + assert.equal(contexts1.length, 1) + assert.equal(contexts1[0].provider.id, 1) + assert.equal((contexts1[0] as any)._dataSetId, 1n) }) it('fails when provided an invalid data set id', async () => { @@ -1009,5 +1010,47 @@ describe('Synapse', () => { } } }) + + it('selects existing data set by default when metadata matches', async () => { + const metadata = { + environment: 'test', + withCDN: '', + } + const contexts = await synapse.storage.createContexts({ + count: 1, + metadata, + }) + assert.equal(contexts.length, 1) + assert.equal(contexts[0].provider.id, 1) + assert.equal((contexts[0] as any)._dataSetId, 1n) + }) + + it('avoids existing data set when provider is excluded even when metadata matches', async () => { + const metadata = { + environment: 'test', + withCDN: '', + } + const contexts = await synapse.storage.createContexts({ + count: 1, + metadata, + excludeProviderIds: [1], + }) + assert.equal(contexts.length, 1) + assert.notEqual(contexts[0].provider.id, 1) + }) + + it('creates new data set context when forced even when metadata matches', async () => { + const metadata = { + environment: 'test', + withCDN: '', + } + const contexts = await synapse.storage.createContexts({ + count: 1, + metadata, + forceCreateDataSets: true, + }) + assert.equal(contexts.length, 1) + assert.equal((contexts[0] as any)._dataSetId, undefined) + }) }) }) diff --git a/packages/synapse-sdk/src/types.ts b/packages/synapse-sdk/src/types.ts index c6716ac15..972fe12bb 100644 --- a/packages/synapse-sdk/src/types.ts +++ b/packages/synapse-sdk/src/types.ts @@ -317,6 +317,8 @@ export interface CreateContextsOptions { * Must be no longer than count */ dataSetIds?: number[] + /** Do not select any of these providers */ + excludeProviderIds?: number[] /** Whether to enable CDN services */ withCDN?: boolean withIpni?: boolean From 178073d3749933df369c57ea959f4ebc05d95239 Mon Sep 17 00:00:00 2001 From: William Morriss Date: Thu, 30 Oct 2025 14:49:23 -0500 Subject: [PATCH 05/28] add broken test --- packages/synapse-sdk/src/storage/context.ts | 2 +- packages/synapse-sdk/src/test/synapse.test.ts | 66 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index 214b69205..741b7e9c6 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -239,7 +239,7 @@ export class StorageContext { } } } - if (resolutions.length <= count) { + if (resolutions.length < count) { const excludeProviderIds = [ ...(options.excludeProviderIds ?? []), ...resolutions.map((resolution) => resolution.provider.id), diff --git a/packages/synapse-sdk/src/test/synapse.test.ts b/packages/synapse-sdk/src/test/synapse.test.ts index e0ed9fe47..379c02513 100644 --- a/packages/synapse-sdk/src/test/synapse.test.ts +++ b/packages/synapse-sdk/src/test/synapse.test.ts @@ -947,6 +947,39 @@ describe('Synapse', () => { assert.equal((contexts[1] as any)._dataSetId, undefined) }) + it('uses existing data set specified by providerId when metadata matches', async () => { + const metadata = { + environment: 'test', + withCDN: '', + } + const contexts = await synapse.storage.createContexts({ + providerIds: [mockProviders[0].providerId], + metadata, + count: 1, + }) + assert.equal(contexts.length, 1) + assert.equal(BigInt(contexts[0].provider.id), mockProviders[0].providerId) + // should use existing data set + assert.equal((contexts[0] as any)._dataSetId, 1n) + }) + + it('force creates new data set specified by providerId even when metadata matches', async () => { + const metadata = { + environment: 'test', + withCDN: '', + } + const contexts = await synapse.storage.createContexts({ + providerIds: [mockProviders[0].providerId], + metadata, + count: 1, + forceCreateDataSets: true, + }) + assert.equal(contexts.length, 1) + assert.equal(BigInt(contexts[0].provider.id), mockProviders[0].providerId) + // should create new data set + assert.equal((contexts[0] as any)._dataSetId, undefined) + }) + it('fails when provided an invalid providerId', async () => { try { await synapse.storage.createContexts({ @@ -970,6 +1003,39 @@ describe('Synapse', () => { assert.equal((contexts[0] as any)._dataSetId, undefined) }) + it('uses existing data set specified by provider address when metadata matches', async () => { + const metadata = { + environment: 'test', + withCDN: '', + } + const contexts = await synapse.storage.createContexts({ + providerAddresses: [mockProviders[0].info.serviceProvider], + metadata, + count: 1, + }) + assert.equal(contexts.length, 1) + assert.equal(BigInt(contexts[0].provider.id), mockProviders[0].providerId) + // should use existing data set + assert.equal((contexts[0] as any)._dataSetId, 1n) + }) + + it('force creates new data set with provider specified address even when metadata matches', async () => { + const metadata = { + environment: 'test', + withCDN: '', + } + const contexts = await synapse.storage.createContexts({ + providerAddresses: [mockProviders[0].info.serviceProvider], + metadata, + count: 1, + forceCreateDataSets: true, + }) + assert.equal(contexts.length, 1) + assert.equal(BigInt(contexts[0].provider.id), mockProviders[0].providerId) + // should create new data sets + assert.equal((contexts[0] as any)._dataSetId, undefined) + }) + it('fails when provided an invalid provider address', async () => { try { await synapse.storage.createContexts({ From 55c324d4a47ca379fb82370374b5d47fdb60d351 Mon Sep 17 00:00:00 2001 From: William Morriss Date: Thu, 30 Oct 2025 15:00:04 -0500 Subject: [PATCH 06/28] fix the test by converting bigint to number --- packages/synapse-sdk/src/storage/context.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index 741b7e9c6..301b592ed 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -515,7 +515,7 @@ export class StorageContext { const providerDataSets = ( dataSets as Awaited> ).filter((ps) => { - if (ps.providerId !== provider.id || !ps.isLive || !ps.isManaged || ps.pdpEndEpoch !== 0) { + if (ps.providerId !== Number(provider.id) || !ps.isLive || !ps.isManaged || ps.pdpEndEpoch !== 0) { return false } // Check if metadata matches From 9511a10ee676e9a6f02b880e70886bc793d1c848 Mon Sep 17 00:00:00 2001 From: William Morriss Date: Thu, 30 Oct 2025 15:05:54 -0500 Subject: [PATCH 07/28] Revert "fix the test by converting bigint to number" This reverts commit 55c324d4a47ca379fb82370374b5d47fdb60d351. --- packages/synapse-sdk/src/storage/context.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index 301b592ed..741b7e9c6 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -515,7 +515,7 @@ export class StorageContext { const providerDataSets = ( dataSets as Awaited> ).filter((ps) => { - if (ps.providerId !== Number(provider.id) || !ps.isLive || !ps.isManaged || ps.pdpEndEpoch !== 0) { + if (ps.providerId !== provider.id || !ps.isLive || !ps.isManaged || ps.pdpEndEpoch !== 0) { return false } // Check if metadata matches From 6afaa65ff18f3d6c6f56878823fba3329c7bd5e5 Mon Sep 17 00:00:00 2001 From: William Morriss Date: Thu, 30 Oct 2025 15:09:24 -0500 Subject: [PATCH 08/28] fix the test by fixing the types --- packages/synapse-sdk/src/test/synapse.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/synapse-sdk/src/test/synapse.test.ts b/packages/synapse-sdk/src/test/synapse.test.ts index 379c02513..68db7b629 100644 --- a/packages/synapse-sdk/src/test/synapse.test.ts +++ b/packages/synapse-sdk/src/test/synapse.test.ts @@ -953,7 +953,7 @@ describe('Synapse', () => { withCDN: '', } const contexts = await synapse.storage.createContexts({ - providerIds: [mockProviders[0].providerId], + providerIds: [mockProviders[0].providerId].map(Number), metadata, count: 1, }) @@ -969,7 +969,7 @@ describe('Synapse', () => { withCDN: '', } const contexts = await synapse.storage.createContexts({ - providerIds: [mockProviders[0].providerId], + providerIds: [mockProviders[0].providerId].map(Number), metadata, count: 1, forceCreateDataSets: true, From 21bb003d69eff680c234252f0ed6d63081c070da Mon Sep 17 00:00:00 2001 From: William Morriss Date: Thu, 30 Oct 2025 15:29:58 -0500 Subject: [PATCH 09/28] it can attempt to create numerous contexts, returning fewer --- packages/synapse-sdk/src/storage/context.ts | 35 ++++++++++++------- packages/synapse-sdk/src/test/synapse.test.ts | 8 +++++ 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index 741b7e9c6..4229bb10c 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -59,6 +59,8 @@ import { import { combineMetadata, metadataMatches, objectToEntries, validatePieceMetadata } from '../utils/metadata.ts' import type { WarmStorageService } from '../warm-storage/index.ts' +const NO_REMAINING_PROVIDERS_ERROR_MESSAGE = 'No approved service providers available' + export class StorageContext { private readonly _synapse: Synapse private readonly _provider: ProviderInfo @@ -245,18 +247,25 @@ export class StorageContext { ...resolutions.map((resolution) => resolution.provider.id), ] for (let i = resolutions.length; i < count; i++) { - const resolution = await StorageContext.smartSelectProvider( - clientAddress, - options.metadata ?? {}, - warmStorageService, - spRegistry, - excludeProviderIds, - options.forceCreateDataSets, - options.withIpni, - options.dev - ) - excludeProviderIds.push(resolution.provider.id) - resolutions.push(resolution) + try { + const resolution = await StorageContext.smartSelectProvider( + clientAddress, + options.metadata ?? {}, + warmStorageService, + spRegistry, + excludeProviderIds, + options.forceCreateDataSets, + options.withIpni, + options.dev + ) + excludeProviderIds.push(resolution.provider.id) + resolutions.push(resolution) + } catch (error) { + if (error?.message.includes(NO_REMAINING_PROVIDERS_ERROR_MESSAGE)) { + break + } + throw error + } } } return await Promise.all( @@ -679,7 +688,7 @@ export class StorageContext { ) if (allProviders.length === 0) { - throw createError('StorageContext', 'smartSelectProvider', 'No approved service providers available') + throw createError('StorageContext', 'smartSelectProvider', NO_REMAINING_PROVIDERS_ERROR_MESSAGE) } // Random selection from all providers diff --git a/packages/synapse-sdk/src/test/synapse.test.ts b/packages/synapse-sdk/src/test/synapse.test.ts index 68db7b629..9da9b53d1 100644 --- a/packages/synapse-sdk/src/test/synapse.test.ts +++ b/packages/synapse-sdk/src/test/synapse.test.ts @@ -1118,5 +1118,13 @@ describe('Synapse', () => { assert.equal(contexts.length, 1) assert.equal((contexts[0] as any)._dataSetId, undefined) }) + + it('can attempt to create numerous contexts, returning fewer', async () => { + const contexts = await synapse.storage.createContexts({ + count: 100, + }) + assert.equal(contexts.length, 2) + assert.notEqual(contexts[0].provider.id, contexts[1].provider.id) + }) }) }) From c821d6fc869cdbbb6d3a832c11f5fe87b756f508 Mon Sep 17 00:00:00 2001 From: William Morriss Date: Thu, 30 Oct 2025 15:33:43 -0500 Subject: [PATCH 10/28] min doc and check Error type --- packages/synapse-sdk/src/storage/context.ts | 5 ++++- packages/synapse-sdk/src/storage/manager.ts | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index 4229bb10c..3df20e692 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -178,6 +178,9 @@ export class StorageContext { this._pdpServer = new PDPServer(authHelper, provider.products.PDP.data.serviceURL) } + /** + * Creates new storage contexts with specified options + */ static async createContexts( synapse: Synapse, warmStorageService: WarmStorageService, @@ -261,7 +264,7 @@ export class StorageContext { excludeProviderIds.push(resolution.provider.id) resolutions.push(resolution) } catch (error) { - if (error?.message.includes(NO_REMAINING_PROVIDERS_ERROR_MESSAGE)) { + if (error instanceof Error && error.message.includes(NO_REMAINING_PROVIDERS_ERROR_MESSAGE)) { break } throw error diff --git a/packages/synapse-sdk/src/storage/manager.ts b/packages/synapse-sdk/src/storage/manager.ts index 76223f14c..6e18a52dd 100644 --- a/packages/synapse-sdk/src/storage/manager.ts +++ b/packages/synapse-sdk/src/storage/manager.ts @@ -231,6 +231,9 @@ export class StorageManager { return await StorageContext.performPreflightCheck(this._warmStorageService, this._synapse.payments, size, withCDN) } + /** + * Creates new storage contexts with specified options + */ async createContexts(options?: CreateContextsOptions): Promise { return await StorageContext.createContexts(this._synapse, this._warmStorageService, { ...options, From 4163bd6bedd6da996e1e348540f5daec82001ef9 Mon Sep 17 00:00:00 2001 From: William Morriss Date: Thu, 30 Oct 2025 16:04:50 -0500 Subject: [PATCH 11/28] add failing test --- packages/synapse-sdk/src/test/synapse.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/synapse-sdk/src/test/synapse.test.ts b/packages/synapse-sdk/src/test/synapse.test.ts index 9da9b53d1..bcedddc5b 100644 --- a/packages/synapse-sdk/src/test/synapse.test.ts +++ b/packages/synapse-sdk/src/test/synapse.test.ts @@ -1077,6 +1077,18 @@ describe('Synapse', () => { } }) + it('does not create multiple contexts for the same data set', async () => { + const contexts = await synapse.storage.createContexts({ + count: 2, + dataSetIds: [1, 1], + }) + assert.equal(contexts.length, 2) + assert.equal((contexts[0] as any)._dataSetId, 1) + assert.notEqual((contexts[0] as any)._dataSetId, (contexts[1] as any)._dataSetId) + // should also use different providers in this case + assert.notEqual(contexts[0].provider.id, contexts[1].provider.id) + }) + it('selects existing data set by default when metadata matches', async () => { const metadata = { environment: 'test', From 44f04889fe781a9eb98d8a4fd3e0cc8998828eb6 Mon Sep 17 00:00:00 2001 From: William Morriss Date: Thu, 30 Oct 2025 16:08:32 -0500 Subject: [PATCH 12/28] dedupe with Set --- packages/synapse-sdk/src/storage/context.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index 3df20e692..0e875de59 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -192,7 +192,7 @@ export class StorageContext { const registryAddress = warmStorageService.getServiceProviderRegistryAddress() const spRegistry = new SPRegistryService(synapse.getProvider(), registryAddress) if (options?.dataSetIds) { - for (const dataSetId of options.dataSetIds) { + for (const dataSetId of new Set(options.dataSetIds)) { const resolution = await StorageContext.resolveByDataSetId( dataSetId, warmStorageService, From cd2e36459b9dc5203aae9499d650f7decb4883f6 Mon Sep 17 00:00:00 2001 From: William Morriss Date: Thu, 30 Oct 2025 16:14:56 -0500 Subject: [PATCH 13/28] add failing test --- packages/synapse-sdk/src/test/synapse.test.ts | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/packages/synapse-sdk/src/test/synapse.test.ts b/packages/synapse-sdk/src/test/synapse.test.ts index bcedddc5b..d856da3ba 100644 --- a/packages/synapse-sdk/src/test/synapse.test.ts +++ b/packages/synapse-sdk/src/test/synapse.test.ts @@ -1077,7 +1077,7 @@ describe('Synapse', () => { } }) - it('does not create multiple contexts for the same data set', async () => { + it('does not create multiple contexts for the same data set from duplicate dataSetIds', async () => { const contexts = await synapse.storage.createContexts({ count: 2, dataSetIds: [1, 1], @@ -1089,6 +1089,21 @@ describe('Synapse', () => { assert.notEqual(contexts[0].provider.id, contexts[1].provider.id) }) + it('does not create multiple contexts for the same data set from duplicate providerIds', async () => { + const metadata = { + environment: 'test', + withCDN: '', + } + const contexts = await synapse.storage.createContexts({ + count: 2, + providerIds: [mockProviders[0].providerId, mockProviders[0].providerId].map(Number), + metadata, + }) + assert.equal(contexts.length, 2) + assert.equal((contexts[0] as any)._dataSetId, 1) + assert.notEqual((contexts[0] as any)._dataSetId, (contexts[1] as any)._dataSetId) + }) + it('selects existing data set by default when metadata matches', async () => { const metadata = { environment: 'test', From ca3b3f3f884842187445efca559d9710d01f5d54 Mon Sep 17 00:00:00 2001 From: William Morriss Date: Thu, 30 Oct 2025 16:30:29 -0500 Subject: [PATCH 14/28] fix test with excludeDataSetIds --- packages/synapse-sdk/src/storage/context.ts | 15 ++++++++++++++- packages/synapse-sdk/src/test/synapse.test.ts | 15 +++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index 0e875de59..1c662e2c5 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -220,6 +220,7 @@ export class StorageContext { options.metadata ?? {}, warmStorageService, spRegistry, + resolutions.map((resolution) => resolution.dataSetId), options.forceCreateDataSets ) resolutions.push(resolution) @@ -235,6 +236,7 @@ export class StorageContext { spRegistry, clientAddress, options.metadata ?? {}, + resolutions.map((resolution) => resolution.dataSetId), options.forceCreateDataSets ) resolutions.push(resolution) @@ -364,6 +366,7 @@ export class StorageContext { requestedMetadata, warmStorageService, spRegistry, + [], options.forceCreateDataSet ) } @@ -376,6 +379,7 @@ export class StorageContext { spRegistry, clientAddress, requestedMetadata, + [], options.forceCreateDataSet ) } @@ -499,6 +503,7 @@ export class StorageContext { requestedMetadata: Record, warmStorageService: WarmStorageService, spRegistry: SPRegistryService, + excludeDataSetIds: number[], forceCreateDataSet?: boolean ): Promise { // Fetch provider (always) and dataSets (only if not forcing) in parallel @@ -527,7 +532,13 @@ export class StorageContext { const providerDataSets = ( dataSets as Awaited> ).filter((ps) => { - if (ps.providerId !== provider.id || !ps.isLive || !ps.isManaged || ps.pdpEndEpoch !== 0) { + if ( + ps.providerId !== provider.id || + !ps.isLive || + !ps.isManaged || + ps.pdpEndEpoch !== 0 || + excludeDataSetIds.includes(Number(ps.dataSetId)) + ) { return false } // Check if metadata matches @@ -571,6 +582,7 @@ export class StorageContext { spRegistry: SPRegistryService, signerAddress: string, requestedMetadata: Record, + excludeDataSetIds: number[], forceCreateDataSet?: boolean ): Promise { // Get provider by address @@ -590,6 +602,7 @@ export class StorageContext { requestedMetadata, warmStorageService, spRegistry, + excludeDataSetIds, forceCreateDataSet ) } diff --git a/packages/synapse-sdk/src/test/synapse.test.ts b/packages/synapse-sdk/src/test/synapse.test.ts index d856da3ba..10c31e10c 100644 --- a/packages/synapse-sdk/src/test/synapse.test.ts +++ b/packages/synapse-sdk/src/test/synapse.test.ts @@ -1104,6 +1104,21 @@ describe('Synapse', () => { assert.notEqual((contexts[0] as any)._dataSetId, (contexts[1] as any)._dataSetId) }) + it('does not create multiple contexts for the same data set from duplicate providerAddresses', async () => { + const metadata = { + environment: 'test', + withCDN: '', + } + const contexts = await synapse.storage.createContexts({ + count: 2, + providerAddresses: [mockProviders[0].info.serviceProvider, mockProviders[0].info.serviceProvider], + metadata, + }) + assert.equal(contexts.length, 2) + assert.equal((contexts[0] as any)._dataSetId, 1) + assert.notEqual((contexts[0] as any)._dataSetId, (contexts[1] as any)._dataSetId) + }) + it('selects existing data set by default when metadata matches', async () => { const metadata = { environment: 'test', From 4d20f3ab02dcc66c690da6bbe7c849413f796e2c Mon Sep 17 00:00:00 2001 From: William Morriss Date: Thu, 30 Oct 2025 16:49:19 -0500 Subject: [PATCH 15/28] doc & test createContext won't return multiple contexts for the same data set --- packages/synapse-sdk/src/storage/context.ts | 1 + packages/synapse-sdk/src/storage/manager.ts | 1 + packages/synapse-sdk/src/test/synapse.test.ts | 21 +++++++++++++++++++ packages/synapse-sdk/src/types.ts | 12 ++++------- 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index 1c662e2c5..061eaeb2d 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -180,6 +180,7 @@ export class StorageContext { /** * Creates new storage contexts with specified options + * Each context correseponds to a different data set */ static async createContexts( synapse: Synapse, diff --git a/packages/synapse-sdk/src/storage/manager.ts b/packages/synapse-sdk/src/storage/manager.ts index 6e18a52dd..7a9feacda 100644 --- a/packages/synapse-sdk/src/storage/manager.ts +++ b/packages/synapse-sdk/src/storage/manager.ts @@ -233,6 +233,7 @@ export class StorageManager { /** * Creates new storage contexts with specified options + * Each context correseponds to a different data set */ async createContexts(options?: CreateContextsOptions): Promise { return await StorageContext.createContexts(this._synapse, this._warmStorageService, { diff --git a/packages/synapse-sdk/src/test/synapse.test.ts b/packages/synapse-sdk/src/test/synapse.test.ts index 10c31e10c..e6648d4a6 100644 --- a/packages/synapse-sdk/src/test/synapse.test.ts +++ b/packages/synapse-sdk/src/test/synapse.test.ts @@ -1078,9 +1078,14 @@ describe('Synapse', () => { }) it('does not create multiple contexts for the same data set from duplicate dataSetIds', async () => { + const metadata = { + environment: 'test', + withCDN: '', + } const contexts = await synapse.storage.createContexts({ count: 2, dataSetIds: [1, 1], + metadata, }) assert.equal(contexts.length, 2) assert.equal((contexts[0] as any)._dataSetId, 1) @@ -1119,6 +1124,22 @@ describe('Synapse', () => { assert.notEqual((contexts[0] as any)._dataSetId, (contexts[1] as any)._dataSetId) }) + it('does not create multiple contexts for a specified data set when providerId also provided', async () => { + const metadata = { + environment: 'test', + withCDN: '', + } + const contexts = await synapse.storage.createContexts({ + count: 2, + dataSetIds: [1, 1], + providerIds: [mockProviders[0].providerId, mockProviders[0].providerId].map(Number), + metadata, + }) + assert.equal(contexts.length, 2) + assert.equal((contexts[0] as any)._dataSetId, 1) + assert.notEqual((contexts[0] as any)._dataSetId, (contexts[1] as any)._dataSetId) + }) + it('selects existing data set by default when metadata matches', async () => { const metadata = { environment: 'test', diff --git a/packages/synapse-sdk/src/types.ts b/packages/synapse-sdk/src/types.ts index 972fe12bb..06e7e87ff 100644 --- a/packages/synapse-sdk/src/types.ts +++ b/packages/synapse-sdk/src/types.ts @@ -301,22 +301,18 @@ export interface StorageContextCallbacks { export interface CreateContextsOptions { /** Number of contexts to create (optional, defaults to 2) */ count?: number + /** + * Specific data set IDs to use + */ + dataSetIds?: number[] /** * Specific provider IDs to use (if not using providerAddresses) - * Must be no longer than count */ providerIds?: number[] /** * Specific provider addresses to use (if not using providerIds) - * Must be no longer than count */ providerAddresses?: string[] - /** - * Specific data set IDs to use - * Cannot be used with provider options - * Must be no longer than count - */ - dataSetIds?: number[] /** Do not select any of these providers */ excludeProviderIds?: number[] /** Whether to enable CDN services */ From b0e4b03e15fd37224be5da2072efbed7217e6ec3 Mon Sep 17 00:00:00 2001 From: William Morriss Date: Thu, 30 Oct 2025 17:20:29 -0500 Subject: [PATCH 16/28] document the priority of the createContext options --- packages/synapse-sdk/src/storage/manager.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/synapse-sdk/src/storage/manager.ts b/packages/synapse-sdk/src/storage/manager.ts index 7a9feacda..51dd92531 100644 --- a/packages/synapse-sdk/src/storage/manager.ts +++ b/packages/synapse-sdk/src/storage/manager.ts @@ -232,8 +232,11 @@ export class StorageManager { } /** - * Creates new storage contexts with specified options - * Each context correseponds to a different data set + * Creates up to `options.count` (default=2) storage contexts corresponding to different data sets + * First, selects data sets specified by `options.dataSetIds` + * The remaining selection prefers existing data sets matching `options.metadata` unless `options.forceCreateDataSets`, + * first using storage providers specified by `options.providerIds` else `options.providerAddresses`, + * and then smart-selecting from the remaining service providers */ async createContexts(options?: CreateContextsOptions): Promise { return await StorageContext.createContexts(this._synapse, this._warmStorageService, { From ecdc24e876ab558422cd0566b7b0df1863b45fcd Mon Sep 17 00:00:00 2001 From: William Morriss Date: Thu, 30 Oct 2025 17:32:18 -0500 Subject: [PATCH 17/28] cleanup createDataSet mocking no longer needed --- packages/synapse-sdk/src/test/synapse.test.ts | 53 ------------------- 1 file changed, 53 deletions(-) diff --git a/packages/synapse-sdk/src/test/synapse.test.ts b/packages/synapse-sdk/src/test/synapse.test.ts index e6648d4a6..601892d8e 100644 --- a/packages/synapse-sdk/src/test/synapse.test.ts +++ b/packages/synapse-sdk/src/test/synapse.test.ts @@ -866,49 +866,11 @@ describe('Synapse', () => { }, ] - const DATA_SET_ID = 7 - - const FAKE_TX_HASH = '0x3816d82cb7a6f5cde23f4d63c0763050d13c6b6dc659d0a7e6eba80b0ec76a18' - - const FAKE_TX = { - hash: FAKE_TX_HASH, - from: ADDRESSES.serviceProvider1, - gas: '0x5208', - value: '0x0', - nonce: '0x444', - input: '0x', - v: '0x01', - r: '0x4e2eef88cc6f2dc311aa3b1c8729b6485bd606960e6ae01522298278932c333a', - s: '0x5d0e08d8ecd6ed8034aa956ff593de9dc1d392e73909ef0c0f828918b58327c9', - } - - const FAKE_RECEIPT = { - ...FAKE_TX, - transactionHash: FAKE_TX_HASH, - transactionIndex: '0x10', - blockHash: '0xb91b7314248aaae06f080ad427dbae78b8c5daf72b2446cf843739aef80c6417', - status: '0x1', - blockNumber: '0x127001', - cumulativeGasUsed: '0x52080', - gasUsed: '0x5208', - logs: [makeDataSetCreatedLog(DATA_SET_ID, 1)], - } - beforeEach(async () => { server.use( JSONRPC({ ...presets.basic, serviceRegistry: mockServiceProviderRegistry(mockProviders, mockPDPProducts), - eth_getTransactionByHash: (params) => { - const hash = params[0] - assert.equal(hash, FAKE_TX_HASH) - return FAKE_TX - }, - eth_getTransactionReceipt: (params) => { - const hash = params[0] - assert.equal(hash, FAKE_TX_HASH) - return FAKE_RECEIPT - }, }) ) synapse = await Synapse.create({ signer }) @@ -917,21 +879,6 @@ describe('Synapse', () => { baseUrl: offering.serviceURL, } server.use(PING(pdpOptions)) - server.use(createDataSetHandler(FAKE_TX_HASH, pdpOptions)) - server.use( - dataSetCreationStatusHandler( - FAKE_TX_HASH, - { - ok: true, - dataSetId: DATA_SET_ID, - createMessageHash: '', - dataSetCreated: true, - service: '', - txStatus: '', - }, - pdpOptions - ) - ) } }) From b6e9064d225d5bcd5417bf58aeba76537bd4ab56 Mon Sep 17 00:00:00 2001 From: William Morriss Date: Thu, 30 Oct 2025 17:49:46 -0500 Subject: [PATCH 18/28] skipProviderIds sooner --- packages/synapse-sdk/src/storage/context.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index 061eaeb2d..e10eb85de 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -629,9 +629,15 @@ export class StorageContext { // Get client's data sets const dataSets = await warmStorageService.getClientDataSetsWithDetails(signerAddress) + const skipProviderIds = new Set(excludeProviderIds) // Filter for managed data sets with matching metadata const managedDataSets = dataSets.filter( - (ps) => ps.isLive && ps.isManaged && ps.pdpEndEpoch === 0 && metadataMatches(ps.metadata, requestedMetadata) + (ps) => + ps.isLive && + ps.isManaged && + ps.pdpEndEpoch === 0 && + metadataMatches(ps.metadata, requestedMetadata) && + !skipProviderIds.has(ps.providerId) ) if (managedDataSets.length > 0 && !forceCreateDataSet) { @@ -644,8 +650,6 @@ export class StorageContext { // Create async generator that yields providers lazily async function* generateProviders(): AsyncGenerator { - const skipProviderIds = new Set(excludeProviderIds) - // First, yield providers from existing data sets (in sorted order) for (const dataSet of sorted) { const provider = await spRegistry.getProvider(dataSet.providerId) From 06eb886b9cd6b0d5ed20847e122740246badb814 Mon Sep 17 00:00:00 2001 From: William Morriss Date: Thu, 30 Oct 2025 17:56:07 -0500 Subject: [PATCH 19/28] skipProviderIds even sooner --- packages/synapse-sdk/src/storage/context.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index e10eb85de..a95dd18b6 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -652,6 +652,9 @@ export class StorageContext { async function* generateProviders(): AsyncGenerator { // First, yield providers from existing data sets (in sorted order) for (const dataSet of sorted) { + if (skipProviderIds.has(dataSet.providerId)) { + continue + } const provider = await spRegistry.getProvider(dataSet.providerId) if (provider == null) { @@ -660,10 +663,8 @@ export class StorageContext { ) continue } - if (!skipProviderIds.has(provider.id)) { - skipProviderIds.add(provider.id) - yield provider - } + skipProviderIds.add(provider.id) + yield provider } } From fe71415e114bc8e87784ad4e8e2fd9cdd822cace Mon Sep 17 00:00:00 2001 From: William Morriss Date: Thu, 30 Oct 2025 18:11:13 -0500 Subject: [PATCH 20/28] test default params --- packages/synapse-sdk/src/test/synapse.test.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/synapse-sdk/src/test/synapse.test.ts b/packages/synapse-sdk/src/test/synapse.test.ts index 601892d8e..d32da32f7 100644 --- a/packages/synapse-sdk/src/test/synapse.test.ts +++ b/packages/synapse-sdk/src/test/synapse.test.ts @@ -1129,6 +1129,14 @@ describe('Synapse', () => { assert.equal((contexts[0] as any)._dataSetId, undefined) }) + it('can select new data sets from different providers using default params', async () => { + const contexts = await synapse.storage.createContexts() + assert.equal(contexts.length, 2) + assert.equal((contexts[0] as any)._dataSetId, undefined) + assert.equal((contexts[1] as any)._dataSetId, undefined) + assert.notEqual(contexts[0].provider.id, contexts[1].provider.id) + }) + it('can attempt to create numerous contexts, returning fewer', async () => { const contexts = await synapse.storage.createContexts({ count: 100, From 2adac93ec87de7925be168d5874ec8f50f6f6b9b Mon Sep 17 00:00:00 2001 From: William Morriss Date: Mon, 3 Nov 2025 12:44:45 -0600 Subject: [PATCH 21/28] rm providerAddresses --- packages/synapse-sdk/src/storage/context.ts | 22 +----- packages/synapse-sdk/src/test/synapse.test.ts | 74 ------------------- packages/synapse-sdk/src/types.ts | 6 +- 3 files changed, 4 insertions(+), 98 deletions(-) diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index a95dd18b6..981c7ecb5 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -185,14 +185,14 @@ export class StorageContext { static async createContexts( synapse: Synapse, warmStorageService: WarmStorageService, - options: CreateContextsOptions = {} + options: CreateContextsOptions ): Promise { const count = options?.count ?? 2 const resolutions: ProviderSelectionResult[] = [] const clientAddress = await synapse.getClient().getAddress() const registryAddress = warmStorageService.getServiceProviderRegistryAddress() const spRegistry = new SPRegistryService(synapse.getProvider(), registryAddress) - if (options?.dataSetIds) { + if (options.dataSetIds) { for (const dataSetId of new Set(options.dataSetIds)) { const resolution = await StorageContext.resolveByDataSetId( dataSetId, @@ -213,7 +213,7 @@ export class StorageContext { } } if (resolutions.length < count) { - if (options?.providerIds) { + if (options.providerIds) { for (const providerId of options.providerIds) { const resolution = await StorageContext.resolveByProviderId( clientAddress, @@ -229,22 +229,6 @@ export class StorageContext { break } } - } else if (options?.providerAddresses) { - for (const providerAddress of options.providerAddresses) { - const resolution = await StorageContext.resolveByProviderAddress( - providerAddress, - warmStorageService, - spRegistry, - clientAddress, - options.metadata ?? {}, - resolutions.map((resolution) => resolution.dataSetId), - options.forceCreateDataSets - ) - resolutions.push(resolution) - if (resolutions.length >= count) { - break - } - } } } if (resolutions.length < count) { diff --git a/packages/synapse-sdk/src/test/synapse.test.ts b/packages/synapse-sdk/src/test/synapse.test.ts index d32da32f7..0b98bffe8 100644 --- a/packages/synapse-sdk/src/test/synapse.test.ts +++ b/packages/synapse-sdk/src/test/synapse.test.ts @@ -938,65 +938,6 @@ describe('Synapse', () => { } }) - it('selects providers specified by address', async () => { - const contexts = await synapse.storage.createContexts({ - providerAddresses: [mockProviders[1].info.serviceProvider, mockProviders[0].info.serviceProvider], - }) - assert.equal(contexts.length, 2) - assert.equal(BigInt(contexts[1].provider.id), mockProviders[0].providerId) - assert.equal(BigInt(contexts[0].provider.id), mockProviders[1].providerId) - // should create new data sets - assert.equal((contexts[1] as any)._dataSetId, undefined) - assert.equal((contexts[0] as any)._dataSetId, undefined) - }) - - it('uses existing data set specified by provider address when metadata matches', async () => { - const metadata = { - environment: 'test', - withCDN: '', - } - const contexts = await synapse.storage.createContexts({ - providerAddresses: [mockProviders[0].info.serviceProvider], - metadata, - count: 1, - }) - assert.equal(contexts.length, 1) - assert.equal(BigInt(contexts[0].provider.id), mockProviders[0].providerId) - // should use existing data set - assert.equal((contexts[0] as any)._dataSetId, 1n) - }) - - it('force creates new data set with provider specified address even when metadata matches', async () => { - const metadata = { - environment: 'test', - withCDN: '', - } - const contexts = await synapse.storage.createContexts({ - providerAddresses: [mockProviders[0].info.serviceProvider], - metadata, - count: 1, - forceCreateDataSets: true, - }) - assert.equal(contexts.length, 1) - assert.equal(BigInt(contexts[0].provider.id), mockProviders[0].providerId) - // should create new data sets - assert.equal((contexts[0] as any)._dataSetId, undefined) - }) - - it('fails when provided an invalid provider address', async () => { - try { - await synapse.storage.createContexts({ - providerAddresses: [ADDRESSES.client1], - }) - assert.fail('Expected createContexts to fail for invalid specified provider address') - } catch (error: any) { - assert.equal( - error?.message, - `StorageContext resolveByProviderAddress failed: Provider ${ADDRESSES.client1} not found in registry` - ) - } - }) - it('selects providers specified by data set id', async () => { const contexts1 = await synapse.storage.createContexts({ count: 1, @@ -1056,21 +997,6 @@ describe('Synapse', () => { assert.notEqual((contexts[0] as any)._dataSetId, (contexts[1] as any)._dataSetId) }) - it('does not create multiple contexts for the same data set from duplicate providerAddresses', async () => { - const metadata = { - environment: 'test', - withCDN: '', - } - const contexts = await synapse.storage.createContexts({ - count: 2, - providerAddresses: [mockProviders[0].info.serviceProvider, mockProviders[0].info.serviceProvider], - metadata, - }) - assert.equal(contexts.length, 2) - assert.equal((contexts[0] as any)._dataSetId, 1) - assert.notEqual((contexts[0] as any)._dataSetId, (contexts[1] as any)._dataSetId) - }) - it('does not create multiple contexts for a specified data set when providerId also provided', async () => { const metadata = { environment: 'test', diff --git a/packages/synapse-sdk/src/types.ts b/packages/synapse-sdk/src/types.ts index 06e7e87ff..42faa2f6c 100644 --- a/packages/synapse-sdk/src/types.ts +++ b/packages/synapse-sdk/src/types.ts @@ -306,13 +306,9 @@ export interface CreateContextsOptions { */ dataSetIds?: number[] /** - * Specific provider IDs to use (if not using providerAddresses) + * Specific provider IDs to use */ providerIds?: number[] - /** - * Specific provider addresses to use (if not using providerIds) - */ - providerAddresses?: string[] /** Do not select any of these providers */ excludeProviderIds?: number[] /** Whether to enable CDN services */ From 95a69ce9cd97db5a4700449cbb4867e751645bdd Mon Sep 17 00:00:00 2001 From: William Morriss Date: Mon, 3 Nov 2025 13:01:27 -0600 Subject: [PATCH 22/28] fix imports --- packages/synapse-sdk/src/test/synapse.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/synapse-sdk/src/test/synapse.test.ts b/packages/synapse-sdk/src/test/synapse.test.ts index 3dec61316..c7cef986d 100644 --- a/packages/synapse-sdk/src/test/synapse.test.ts +++ b/packages/synapse-sdk/src/test/synapse.test.ts @@ -4,8 +4,8 @@ * Basic tests for Synapse class */ -import type { PDPOffering } from '@filoz/synapse-core/utils' import { capabilitiesListToObject, encodePDPCapabilities } from '@filoz/synapse-core/utils' +import type { PDPOffering } from '@filoz/synapse-core/warm-storage' import { assert } from 'chai' import { ethers } from 'ethers' import { setup } from 'iso-web/msw' From 21aa5e2c5c41f8d36f1175d3d493f7e46410a307 Mon Sep 17 00:00:00 2001 From: William Morriss Date: Mon, 3 Nov 2025 15:09:02 -0600 Subject: [PATCH 23/28] async selection where possible --- packages/synapse-sdk/src/storage/context.ts | 62 +++++++++------------ 1 file changed, 26 insertions(+), 36 deletions(-) diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index 981c7ecb5..509e62e07 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -193,49 +193,50 @@ export class StorageContext { const registryAddress = warmStorageService.getServiceProviderRegistryAddress() const spRegistry = new SPRegistryService(synapse.getProvider(), registryAddress) if (options.dataSetIds) { + const selections = [] for (const dataSetId of new Set(options.dataSetIds)) { - const resolution = await StorageContext.resolveByDataSetId( - dataSetId, - warmStorageService, - spRegistry, - clientAddress, - { + selections.push( + StorageContext.resolveByDataSetId(dataSetId, warmStorageService, spRegistry, clientAddress, { withCDN: options.withCDN, withIpni: options.withIpni, dev: options.dev, metadata: options.metadata, - } + }) ) - resolutions.push(resolution) - if (resolutions.length >= count) { + if (selections.length >= count) { break } } + resolutions.push(...(await Promise.all(selections))) } + const resolvedProviderIds = resolutions.map((resolution) => resolution.provider.id) if (resolutions.length < count) { if (options.providerIds) { - for (const providerId of options.providerIds) { - const resolution = await StorageContext.resolveByProviderId( - clientAddress, - providerId, - options.metadata ?? {}, - warmStorageService, - spRegistry, - resolutions.map((resolution) => resolution.dataSetId), - options.forceCreateDataSets + const selections = [] + // NOTE: Set.difference is unavailable in some targets + for (const providerId of [...new Set(options.providerIds)].filter( + (providerId) => !resolvedProviderIds.includes(providerId) + )) { + selections.push( + StorageContext.resolveByProviderId( + clientAddress, + providerId, + options.metadata ?? {}, + warmStorageService, + spRegistry, + options.forceCreateDataSets + ) ) - resolutions.push(resolution) - if (resolutions.length >= count) { + resolvedProviderIds.push(providerId) + if (selections.length + resolutions.length >= count) { break } } + resolutions.push(...(await Promise.all(selections))) } } if (resolutions.length < count) { - const excludeProviderIds = [ - ...(options.excludeProviderIds ?? []), - ...resolutions.map((resolution) => resolution.provider.id), - ] + const excludeProviderIds = [...(options.excludeProviderIds ?? []), ...resolvedProviderIds] for (let i = resolutions.length; i < count; i++) { try { const resolution = await StorageContext.smartSelectProvider( @@ -351,7 +352,6 @@ export class StorageContext { requestedMetadata, warmStorageService, spRegistry, - [], options.forceCreateDataSet ) } @@ -364,7 +364,6 @@ export class StorageContext { spRegistry, clientAddress, requestedMetadata, - [], options.forceCreateDataSet ) } @@ -488,7 +487,6 @@ export class StorageContext { requestedMetadata: Record, warmStorageService: WarmStorageService, spRegistry: SPRegistryService, - excludeDataSetIds: number[], forceCreateDataSet?: boolean ): Promise { // Fetch provider (always) and dataSets (only if not forcing) in parallel @@ -517,13 +515,7 @@ export class StorageContext { const providerDataSets = ( dataSets as Awaited> ).filter((ps) => { - if ( - ps.providerId !== provider.id || - !ps.isLive || - !ps.isManaged || - ps.pdpEndEpoch !== 0 || - excludeDataSetIds.includes(Number(ps.dataSetId)) - ) { + if (ps.providerId !== provider.id || !ps.isLive || !ps.isManaged || ps.pdpEndEpoch !== 0) { return false } // Check if metadata matches @@ -567,7 +559,6 @@ export class StorageContext { spRegistry: SPRegistryService, signerAddress: string, requestedMetadata: Record, - excludeDataSetIds: number[], forceCreateDataSet?: boolean ): Promise { // Get provider by address @@ -587,7 +578,6 @@ export class StorageContext { requestedMetadata, warmStorageService, spRegistry, - excludeDataSetIds, forceCreateDataSet ) } From 85299bf5724a081b19c248bfc6d2a5a778c74b41 Mon Sep 17 00:00:00 2001 From: William Morriss Date: Mon, 3 Nov 2025 16:27:21 -0600 Subject: [PATCH 24/28] only check each provider once --- packages/synapse-sdk/src/storage/context.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index 509e62e07..0915f09fe 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -629,6 +629,7 @@ export class StorageContext { if (skipProviderIds.has(dataSet.providerId)) { continue } + skipProviderIds.add(dataSet.providerId) const provider = await spRegistry.getProvider(dataSet.providerId) if (provider == null) { @@ -637,7 +638,6 @@ export class StorageContext { ) continue } - skipProviderIds.add(provider.id) yield provider } } From 2e65e15ba6e13c523a5e425ef6d3a37f3cb1dd7a Mon Sep 17 00:00:00 2001 From: William Morriss Date: Mon, 3 Nov 2025 17:09:08 -0600 Subject: [PATCH 25/28] nonnull these args earlier --- packages/synapse-sdk/src/storage/context.ts | 30 ++++++++++----------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index 0915f09fe..90b98e61a 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -245,9 +245,9 @@ export class StorageContext { warmStorageService, spRegistry, excludeProviderIds, - options.forceCreateDataSets, - options.withIpni, - options.dev + options.forceCreateDataSets ?? false, + options.withIpni ?? false, + options.dev ?? false ) excludeProviderIds.push(resolution.provider.id) resolutions.push(resolution) @@ -374,10 +374,10 @@ export class StorageContext { requestedMetadata, warmStorageService, spRegistry, - options.excludeProviderIds, - options.forceCreateDataSet, - options.withIpni, - options.dev + options.excludeProviderIds ?? [], + options.forceCreateDataSet ?? false, + options.withIpni ?? false, + options.dev ?? false ) } @@ -591,10 +591,10 @@ export class StorageContext { requestedMetadata: Record, warmStorageService: WarmStorageService, spRegistry: SPRegistryService, - excludeProviderIds?: number[], - forceCreateDataSet?: boolean, - withIpni?: boolean, - dev?: boolean + excludeProviderIds: number[], + forceCreateDataSet: boolean, + withIpni: boolean, + dev: boolean ): Promise { // Strategy: // 1. Try to find existing data sets first @@ -707,8 +707,8 @@ export class StorageContext { */ private static async selectRandomProvider( providers: ProviderInfo[], - withIpni?: boolean, - dev?: boolean + withIpni: boolean, + dev: boolean ): Promise { if (providers.length === 0) { throw createError('StorageContext', 'selectRandomProvider', 'No providers available') @@ -740,10 +740,10 @@ export class StorageContext { */ private static async selectProviderWithPing( providers: AsyncIterable, - options?: { withIpni?: boolean; dev?: boolean } + options: { withIpni: boolean; dev: boolean } ): Promise { let providerCount = 0 - const { withIpni, dev } = options ?? {} + const { withIpni, dev } = options // Try providers in order until we find one that responds to ping for await (const provider of providers) { From e36fca43121a81f92b16a6850b9e18c1ed79b616 Mon Sep 17 00:00:00 2001 From: William Morriss Date: Mon, 3 Nov 2025 17:25:15 -0600 Subject: [PATCH 26/28] filter by dev and ipni before selectProviderWithPing async generator --- packages/synapse-sdk/src/storage/context.ts | 46 ++++++++------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index 90b98e61a..e8eab18ce 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -638,15 +638,21 @@ export class StorageContext { ) continue } + + if (withIpni && provider.products.PDP?.data.ipniIpfs === false) { + continue + } + + if (!dev && provider.products.PDP?.capabilities?.dev != null) { + continue + } + yield provider } } try { - const selectedProvider = await StorageContext.selectProviderWithPing(generateProviders(), { - dev, - withIpni, - }) + const selectedProvider = await StorageContext.selectProviderWithPing(generateProviders()) // Find the first matching data set ID for this provider // Match by provider ID (stable identifier in the registry) @@ -680,7 +686,10 @@ export class StorageContext { const approvedIds = await warmStorageService.getApprovedProviderIds() const approvedProviders = await spRegistry.getProviders(approvedIds) const allProviders = approvedProviders.filter( - (provider: ProviderInfo) => excludeProviderIds?.includes(provider.id) !== true + (provider: ProviderInfo) => + (!withIpni || provider.products.PDP?.data.ipniIpfs === true) && + (dev || provider.products.PDP?.capabilities?.dev == null) && + !excludeProviderIds.includes(provider.id) ) if (allProviders.length === 0) { @@ -688,7 +697,7 @@ export class StorageContext { } // Random selection from all providers - const provider = await StorageContext.selectRandomProvider(allProviders, withIpni, dev) + const provider = await StorageContext.selectRandomProvider(allProviders) return { provider, @@ -705,11 +714,7 @@ export class StorageContext { * @param dev - Include dev providers * @returns Selected provider */ - private static async selectRandomProvider( - providers: ProviderInfo[], - withIpni: boolean, - dev: boolean - ): Promise { + private static async selectRandomProvider(providers: ProviderInfo[]): Promise { if (providers.length === 0) { throw createError('StorageContext', 'selectRandomProvider', 'No providers available') } @@ -725,10 +730,7 @@ export class StorageContext { } } - return await StorageContext.selectProviderWithPing(generateRandomProviders(), { - withIpni, - dev, - }) + return await StorageContext.selectProviderWithPing(generateRandomProviders()) } /** @@ -738,23 +740,11 @@ export class StorageContext { * @returns The first provider that responds * @throws If all providers fail */ - private static async selectProviderWithPing( - providers: AsyncIterable, - options: { withIpni: boolean; dev: boolean } - ): Promise { + private static async selectProviderWithPing(providers: AsyncIterable): Promise { let providerCount = 0 - const { withIpni, dev } = options // Try providers in order until we find one that responds to ping for await (const provider of providers) { - if (withIpni && provider?.products.PDP?.data.ipniIpfs === false) { - continue - } - - if (!dev && provider?.products.PDP?.capabilities?.dev != null) { - continue - } - providerCount++ try { // Create a temporary PDPServer for this specific provider's endpoint From fb23990166c89758c21c539eff7d0c68a0c3cbc1 Mon Sep 17 00:00:00 2001 From: William Morriss Date: Mon, 3 Nov 2025 18:25:13 -0600 Subject: [PATCH 27/28] rewrite doc for synapse.storage.createContexts --- packages/synapse-sdk/src/storage/manager.ts | 28 +++++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/packages/synapse-sdk/src/storage/manager.ts b/packages/synapse-sdk/src/storage/manager.ts index 3783fa34a..e940f73a4 100644 --- a/packages/synapse-sdk/src/storage/manager.ts +++ b/packages/synapse-sdk/src/storage/manager.ts @@ -232,11 +232,29 @@ export class StorageManager { } /** - * Creates up to `options.count` (default=2) storage contexts corresponding to different data sets - * First, selects data sets specified by `options.dataSetIds` - * The remaining selection prefers existing data sets matching `options.metadata` unless `options.forceCreateDataSets`, - * first using storage providers specified by `options.providerIds` else `options.providerAddresses`, - * and then smart-selecting from the remaining service providers + * Creates storage contexts for multi-provider storage deals and other operations. + * + * By storing data with multiple independent providers, you reduce dependency on any + * single provider and improve overall data availability. Use contexts together as a group. + * + * Contexts are selected by priority: + * 1. Specified datasets (`dataSetIds`) - uses their existing providers + * 2. Specified providers (`providerIds` or `providerAddresses`) - finds or creates matching datasets + * 3. Automatically selected from remaining approved providers + * + * For automatic selection, existing datasets matching the `metadata` are reused unless + * `forceCreateDataSets` is true. Providers are randomly chosen to distribute across the network. + * + * @param synapse - Synapse instance + * @param warmStorageService - Warm storage service instance + * @param options - Configuration options + * @param options.count - Maximum number of contexts to create (default: 2) + * @param options.dataSetIds - Specific dataset IDs to include + * @param options.providerIds - Specific provider IDs to use + * @param options.metadata - Metadata to match when finding/creating datasets + * @param options.forceCreateDataSets - Always create new datasets instead of reusing existing ones + * @param options.excludeProviderIds - Provider IDs to skip during selection + * @returns Promise resolving to array of storage contexts */ async createContexts(options?: CreateContextsOptions): Promise { return await StorageContext.createContexts(this._synapse, this._warmStorageService, { From 877f4ed0cd073e082e12c3cf56f2e8159b200179 Mon Sep 17 00:00:00 2001 From: William Morriss Date: Mon, 3 Nov 2025 18:26:30 -0600 Subject: [PATCH 28/28] Update packages/synapse-sdk/src/storage/context.ts Co-authored-by: Hugo Dias --- packages/synapse-sdk/src/storage/context.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index e8eab18ce..57224a087 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -180,7 +180,7 @@ export class StorageContext { /** * Creates new storage contexts with specified options - * Each context correseponds to a different data set + * Each context corresponds to a different data set */ static async createContexts( synapse: Synapse,