diff --git a/packages/synapse-core/src/utils/rand.ts b/packages/synapse-core/src/utils/rand.ts index e682615ac..fdd9f39aa 100644 --- a/packages/synapse-core/src/utils/rand.ts +++ b/packages/synapse-core/src/utils/rand.ts @@ -30,18 +30,3 @@ export function randU256(): bigint { export function fallbackRandIndex(length: number): number { return Math.floor(Math.random() * length) } - -/** - * Provides a random index into an array of supplied length (0 <= index < length) - * @param length - exclusive upper boundary - * @returns a valid index - */ -export function randIndex(length: number): number { - if (crypto?.getRandomValues != null) { - const randomBytes = new Uint32Array(1) - crypto.getRandomValues(randomBytes) - return randomBytes[0] % length - } else { - return fallbackRandIndex(length) - } -} diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index 57224a087..847899955 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -23,7 +23,7 @@ */ import * as SP from '@filoz/synapse-core/sp' -import { randIndex, randU256 } from '@filoz/synapse-core/utils' +import { randU256 } from '@filoz/synapse-core/utils' import type { ethers } from 'ethers' import type { Hex } from 'viem' import type { PaymentsService } from '../payments/index.ts' @@ -31,7 +31,7 @@ import { PDPAuthHelper, PDPServer } from '../pdp/index.ts' import { PDPVerifier } from '../pdp/verifier.ts' import { asPieceCID } from '../piece/index.ts' import { SPRegistryService } from '../sp-registry/index.ts' -import type { ProviderInfo } from '../sp-registry/types.ts' +import type { ProviderInfo, ServiceProduct } from '../sp-registry/types.ts' import type { Synapse } from '../synapse.ts' import type { CreateContextsOptions, @@ -605,7 +605,7 @@ export class StorageContext { const skipProviderIds = new Set(excludeProviderIds) // Filter for managed data sets with matching metadata - const managedDataSets = dataSets.filter( + const managedDataSets: EnhancedDataSetInfo[] = dataSets.filter( (ps) => ps.isLive && ps.isManaged && @@ -615,48 +615,38 @@ export class StorageContext { ) if (managedDataSets.length > 0 && !forceCreateDataSet) { - // Prefer data sets with pieces, sort by ID (older first) - const sorted = managedDataSets.sort((a, b) => { - if (a.currentPieceCount > 0 && b.currentPieceCount === 0) return -1 - if (b.currentPieceCount > 0 && a.currentPieceCount === 0) return 1 - return a.pdpVerifierDataSetId - b.pdpVerifierDataSetId - }) - - // Create async generator that yields providers lazily - async function* generateProviders(): AsyncGenerator { - // First, yield providers from existing data sets (in sorted order) - for (const dataSet of sorted) { - if (skipProviderIds.has(dataSet.providerId)) { - continue - } - skipProviderIds.add(dataSet.providerId) - const provider = await spRegistry.getProvider(dataSet.providerId) - - if (provider == null) { - console.warn( - `Provider ID ${dataSet.providerId} for data set ${dataSet.pdpVerifierDataSetId} is not currently approved` - ) - continue - } + // Prefer data sets with pieces + const [hasNoPieces, hasPieces] = managedDataSets + .reduce<[Set, Set]>( + (results: [Set, Set], managedDataSet: EnhancedDataSetInfo) => { + results[managedDataSet.currentPieceCount > 0 ? 1 : 0].add(managedDataSet) + return results + }, + [new Set(), new Set()] + ) + .map((deduped) => [...deduped]) - if (withIpni && provider.products.PDP?.data.ipniIpfs === false) { - continue - } + for (const managedDataSets of [hasPieces, hasNoPieces]) { + const providers: ProviderInfo[] = ( + await Promise.all( + managedDataSets.map((dataSet: EnhancedDataSetInfo) => spRegistry.getProvider(dataSet.providerId)) + ) + ).filter( + (provider: ProviderInfo | null): provider is ProviderInfo => + provider !== null && + (!withIpni || provider.products.PDP?.data.ipniIpfs !== false) && + (dev || provider.products.PDP?.capabilities?.dev == null) + ) - if (!dev && provider.products.PDP?.capabilities?.dev != null) { - continue - } + const selectedProvider = await StorageContext.selectProviderWithPing(providers) - yield provider + if (selectedProvider == null) { + continue } - } - - try { - const selectedProvider = await StorageContext.selectProviderWithPing(generateProviders()) // Find the first matching data set ID for this provider // Match by provider ID (stable identifier in the registry) - const matchingDataSet = sorted.find((ps) => ps.providerId === selectedProvider.id) + const matchingDataSet = managedDataSets.find((ps: EnhancedDataSetInfo) => ps.providerId === selectedProvider.id) if (matchingDataSet == null) { console.warn( @@ -675,10 +665,8 @@ export class StorageContext { dataSetMetadata, } } - } catch (_error) { - console.warn('All providers from existing data sets failed health check. Falling back to all providers.') - // Fall through to select from all approved providers below } + console.warn('All providers from existing data sets failed health check. Falling back to all providers.') } // No existing data sets - select from all approved providers. First we get approved IDs from @@ -696,8 +684,16 @@ export class StorageContext { throw createError('StorageContext', 'smartSelectProvider', NO_REMAINING_PROVIDERS_ERROR_MESSAGE) } - // Random selection from all providers - const provider = await StorageContext.selectRandomProvider(allProviders) + // Select from all providers + const provider = await StorageContext.selectProviderWithPing(allProviders) + + if (provider == null) { + throw createError( + 'StorageContext', + 'selectProviderWithPing', + `All ${allProviders.length} providers failed health check. Storage may be temporarily unavailable.` + ) + } return { provider, @@ -708,72 +704,41 @@ export class StorageContext { } /** - * Select a random provider from a list with ping validation - * @param providers - Array of providers to select from - * @param withIpni - Filter for IPNI support - * @param dev - Include dev providers - * @returns Selected provider + * Select a provider with ping validation. + * @param providers - providers to try + * @returns The first provider that responds, or null if none do */ - private static async selectRandomProvider(providers: ProviderInfo[]): Promise { - if (providers.length === 0) { - throw createError('StorageContext', 'selectRandomProvider', 'No providers available') - } - - // Create async generator that yields providers in random order - async function* generateRandomProviders(): AsyncGenerator { - const remaining = [...providers] - - while (remaining.length > 0) { - // Remove and yield the selected provider - const selected = remaining.splice(randIndex(remaining.length), 1)[0] - yield selected + private static async selectProviderWithPing(providers: ProviderInfo[]): Promise { + type ProviderWithPDP = ProviderInfo & { + products: { + PDP: ServiceProduct } } - return await StorageContext.selectProviderWithPing(generateRandomProviders()) - } - - /** - * Select a provider from an async iterator with ping validation. - * This is shared logic used by both smart selection and random selection. - * @param providers - Async iterable of providers to try - * @returns The first provider that responds - * @throws If all providers fail - */ - private static async selectProviderWithPing(providers: AsyncIterable): Promise { - let providerCount = 0 - - // Try providers in order until we find one that responds to ping - for await (const provider of providers) { - providerCount++ + function hasPDP(provider: ProviderInfo): provider is ProviderWithPDP { + return provider.products.PDP != null + } + // Ping all providers + const pings = providers.filter(hasPDP).map((provider, index) => + new PDPServer(null, provider.products.PDP.data.serviceURL).ping().then( + () => Promise.resolve(provider), + (error) => Promise.reject({ error, index, provider }) + ) + ) + let remaining = pings.length + while (remaining-- > 0) { try { - // Create a temporary PDPServer for this specific provider's endpoint - if (!provider.products.PDP?.data.serviceURL) { - // Skip providers without PDP products - continue - } - const providerPdpServer = new PDPServer(null, provider.products.PDP.data.serviceURL) - await providerPdpServer.ping() - return provider - } catch (error) { + return await Promise.race(pings) + } catch (err: any) { + const { error, index, provider } = err console.warn( `Provider ${provider.serviceProvider} failed ping test:`, error instanceof Error ? error.message : String(error) ) - // Continue to next provider + pings[index] = new Promise(() => undefined) } } - - // All providers failed ping test - if (providerCount === 0) { - throw createError('StorageContext', 'selectProviderWithPing', 'No providers available to select from') - } - - throw createError( - 'StorageContext', - 'selectProviderWithPing', - `All ${providerCount} providers failed health check. Storage may be temporarily unavailable.` - ) + return null } /** diff --git a/packages/synapse-sdk/src/test/rand.test.ts b/packages/synapse-sdk/src/test/rand.test.ts index 09d4f1683..88bc81c72 100644 --- a/packages/synapse-sdk/src/test/rand.test.ts +++ b/packages/synapse-sdk/src/test/rand.test.ts @@ -1,52 +1,49 @@ /* globals describe it */ -import { fallbackRandIndex, fallbackRandU256, randIndex, randU256 } from '@filoz/synapse-core/utils' +import { fallbackRandIndex, fallbackRandU256, randU256 } from '@filoz/synapse-core/utils' import { assert } from 'chai' -const randIndexMethods = [randIndex, fallbackRandIndex] -randIndexMethods.forEach((randIndexMethod) => { - describe(randIndexMethod.name, () => { - it('should return 0 for length 1', () => { - for (let i = 0; i < 32; i++) { - assert.equal(0, randIndexMethod(1)) - } - }) - it('returns both 0 and 1 for length 2', () => { - const counts = [0, 0] - for (let i = 0; i < 32; i++) { - counts[randIndexMethod(counts.length)]++ - } - // this test can fail probabilistically but the probability is low - // each bit should be independent with 50% likelihood - // the probability of getting the same index N times is 2**(1-N) - // so if this test fails, the 50% assumption is likely wrong - assert.isAtLeast(counts[0], 1) - assert.isAtLeast(counts[1], 1) - }) - it('has at least 10 random bits', () => { - const counts = [] - for (let i = 0; i < 10; i++) { - counts.push([0, 0]) - } - for (let i = 0; i < 32; i++) { - let index = randIndexMethod(1024) - assert.isAtLeast(index, 0) - assert.isAtMost(index, 1023) - for (let j = 0; j < 10; j++) { - counts[j][index & 1]++ - index >>= 1 - } - assert.equal(index, 0) - } - // this test can fail probabilistically but the probability is low - // each bit should be independent with 50% likelihood - // the probability of getting the same bitvalue N times is 2**(1-N) - // so if this test fails, the 50% assumption is likely wrong - for (let i = 0; i < 10; i++) { - assert.isAtLeast(counts[i][0], 1) - assert.isAtLeast(counts[i][1], 1) +describe('fallbackRandIndex', () => { + it('should return 0 for length 1', () => { + for (let i = 0; i < 32; i++) { + assert.equal(0, fallbackRandIndex(1)) + } + }) + it('returns both 0 and 1 for length 2', () => { + const counts = [0, 0] + for (let i = 0; i < 32; i++) { + counts[fallbackRandIndex(counts.length)]++ + } + // this test can fail probabilistically but the probability is low + // each bit should be independent with 50% likelihood + // the probability of getting the same index N times is 2**(1-N) + // so if this test fails, the 50% assumption is likely wrong + assert.isAtLeast(counts[0], 1) + assert.isAtLeast(counts[1], 1) + }) + it('has at least 10 random bits', () => { + const counts = [] + for (let i = 0; i < 10; i++) { + counts.push([0, 0]) + } + for (let i = 0; i < 32; i++) { + let index = fallbackRandIndex(1024) + assert.isAtLeast(index, 0) + assert.isAtMost(index, 1023) + for (let j = 0; j < 10; j++) { + counts[j][index & 1]++ + index >>= 1 } - }) + assert.equal(index, 0) + } + // this test can fail probabilistically but the probability is low + // each bit should be independent with 50% likelihood + // the probability of getting the same bitvalue N times is 2**(1-N) + // so if this test fails, the 50% assumption is likely wrong + for (let i = 0; i < 10; i++) { + assert.isAtLeast(counts[i][0], 1) + assert.isAtLeast(counts[i][1], 1) + } }) }) diff --git a/packages/synapse-sdk/src/test/storage.test.ts b/packages/synapse-sdk/src/test/storage.test.ts index f4d6e06e0..3072e9fc5 100644 --- a/packages/synapse-sdk/src/test/storage.test.ts +++ b/packages/synapse-sdk/src/test/storage.test.ts @@ -2392,7 +2392,7 @@ describe('StorageService', () => { }) describe('Provider Ping Validation', () => { - describe('selectRandomProvider with ping validation', () => { + describe('selectProviderWithPing', () => { it('should select first provider that responds to ping', async () => { const testProviders: ProviderInfo[] = [ createSimpleProvider({ @@ -2428,7 +2428,7 @@ describe('StorageService', () => { } try { - const result = await (StorageContext as any).selectRandomProvider(testProviders) + const result = await (StorageContext as any).selectProviderWithPing(testProviders) // Should have selected the second provider (first one failed ping) assert.equal(result.serviceProvider, testProviders[1].serviceProvider) @@ -2438,8 +2438,6 @@ describe('StorageService', () => { } }) - // Test removed: selectRandomProvider no longer supports exclusion functionality - it('should throw error when all providers fail ping', async () => { const testProviders: ProviderInfo[] = [ createSimpleProvider({ @@ -2463,11 +2461,8 @@ describe('StorageService', () => { } try { - await (StorageContext as any).selectRandomProvider(testProviders) - assert.fail('Should have thrown error') - } catch (error: any) { - assert.include(error.message, 'StorageContext selectProviderWithPing failed') - assert.include(error.message, 'All 2 providers failed health check') + const provider = await (StorageContext as any).selectProviderWithPing(testProviders) + assert.isNull(provider) } finally { global.fetch = originalFetch }