diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bc9d25f..02d9cdb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,7 +43,7 @@ jobs: run: dotnet build SlimVector.slnx -c Release --no-restore - name: Run tests - run: dotnet test SlimVector.slnx -c Release --no-build --verbosity normal + run: dotnet test --solution SlimVector.slnx -c Release --no-build --verbosity normal version: name: Compute release channel and version diff --git a/AGENTS.md b/AGENTS.md index bdead56..d9d27f8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -72,11 +72,11 @@ et les performances priment sur la commodité d’une modification locale. Exemples : ```bash -dotnet test tests/SlimVector.Studio.Tests/SlimVector.Studio.Tests.csproj \ +dotnet test --project tests/SlimVector.Studio.Tests/SlimVector.Studio.Tests.csproj \ --no-restore --filter "FullyQualifiedName~StudioIntegrationTests" dotnet build SlimVector.slnx --no-restore -dotnet test SlimVector.slnx --no-restore --no-build +dotnet test --solution SlimVector.slnx --no-restore --no-build ``` ## Benchmarks : toujours mesurer @@ -148,7 +148,7 @@ Avant de conclure : 2. Les tests ciblés passent. 3. Le benchmark pertinent a été rejoué et ses résultats ont été examinés. 4. `dotnet build SlimVector.slnx --no-restore` passe sans avertissement. -5. `dotnet test SlimVector.slnx --no-restore --no-build` passe. +5. `dotnet test --solution SlimVector.slnx --no-restore --no-build` passe. 6. `git diff --check` ne signale aucune erreur. 7. Aucun serveur, processus, fichier temporaire ou artefact de test n’est laissé. 8. Le compte rendu cite les fichiers modifiés, les tests exécutés, les benchmarks diff --git a/Directory.Packages.props b/Directory.Packages.props index 86616ac..273bef3 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -6,24 +6,25 @@ - + - - - - - - - - + + + + + + + + + - - - - + + + + - - + + diff --git a/Dockerfile b/Dockerfile index e101f08..5e4dbb0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,13 @@ # syntax=docker/dockerfile:1.7 -FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build +FROM mcr.microsoft.com/dotnet/sdk:10.0.300 AS build WORKDIR /src RUN apt-get -o Acquire::Retries=5 update \ && apt-get -o Acquire::Retries=5 install -y --no-install-recommends clang llvm zlib1g-dev \ && rm -rf /var/lib/apt/lists/* COPY . . ARG TARGETARCH -RUN arch="$TARGETARCH"; \ +RUN set -eu; \ + arch="$TARGETARCH"; \ if [ "$arch" = "amd64" ]; then arch="x64"; fi; \ rid="linux-$arch"; \ dotnet restore src/SlimVector.Api/SlimVector.Api.csproj -r "$rid"; \ diff --git a/Dockerfile.Studio b/Dockerfile.Studio index 8bfe59a..0f9c233 100644 --- a/Dockerfile.Studio +++ b/Dockerfile.Studio @@ -1,9 +1,10 @@ # syntax=docker/dockerfile:1.7 -FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build +FROM mcr.microsoft.com/dotnet/sdk:10.0.300 AS build WORKDIR /src COPY . . ARG TARGETARCH -RUN arch="$TARGETARCH"; \ +RUN set -eu; \ + arch="$TARGETARCH"; \ if [ "$arch" = "amd64" ]; then arch="x64"; fi; \ rid="linux-$arch"; \ dotnet restore src/SlimVector.Studio/SlimVector.Studio.csproj -r "$rid"; \ diff --git a/README.md b/README.md index 1ed851b..464df46 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,7 @@ JSON is the default public wire format. For vector-heavy client traffic, constru ```bash dotnet format SlimVector.slnx --verify-no-changes dotnet build SlimVector.slnx -c Release -dotnet test SlimVector.slnx -c Release +dotnet test --solution SlimVector.slnx -c Release dotnet publish src/SlimVector.Api/SlimVector.Api.csproj -c Release -r linux-x64 --self-contained true ``` diff --git a/docs/testing.md b/docs/testing.md index 0a4e331..d435483 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -7,7 +7,7 @@ Run deterministic quality gates from the repository root: ```bash dotnet format SlimVector.slnx --verify-no-changes dotnet build SlimVector.slnx -c Release -dotnet test SlimVector.slnx -c Release +dotnet test --solution SlimVector.slnx -c Release ``` The test projects cover domain validation; SIMD Flat, HNSW, IVF-Flat, IVF-PQ, scalar quantization and SSD DiskANN persistence/recall/mutation/rollback; Auto selection, online migration, failed-candidate isolation and restart-safe rollback; BM25, metadata and hybrid fusion; immutable storage/crash reconciliation/compaction; CRUD, eviction, batching, backpressure and reserved/fair token buckets; backup/S3/restore; actual TCP Raft election/failover/snapshot/partition/multi-group plus fourth-member warm-up, restart and removal; geographic outage/restart/divergence; and HTTP/client/admin/429/redirect behavior. diff --git a/global.json b/global.json index f7772fe..1584fd0 100644 --- a/global.json +++ b/global.json @@ -3,5 +3,8 @@ "version": "10.0.300", "rollForward": "latestPatch", "allowPrerelease": false + }, + "test": { + "runner": "Microsoft.Testing.Platform" } } diff --git a/src/SlimVector.Raft/MultiRaftNode.cs b/src/SlimVector.Raft/MultiRaftNode.cs index c654b1d..b41ae8e 100644 --- a/src/SlimVector.Raft/MultiRaftNode.cs +++ b/src/SlimVector.Raft/MultiRaftNode.cs @@ -10,6 +10,8 @@ public sealed class MultiRaftNode : IAsyncDisposable private readonly ConcurrentDictionary _groups = new(StringComparer.Ordinal); private readonly Func _applierFactory; private readonly SemaphoreSlim _lifecycleGate = new(1, 1); + private readonly object _disposeLock = new(); + private Task? _disposeTask; private volatile bool _started; public MultiRaftNode( @@ -226,7 +228,15 @@ public IReadOnlyList GetMembershipStatuses() => _groups .Select(static group => new ClusterMembershipStatus(group.GroupId, group.GetMemberStatuses(), null, null)) .ToArray(); - public async ValueTask DisposeAsync() + public ValueTask DisposeAsync() + { + lock (_disposeLock) + { + return new ValueTask(_disposeTask ??= DisposeCoreAsync()); + } + } + + private async Task DisposeCoreAsync() { await _lifecycleGate.WaitAsync().ConfigureAwait(false); try diff --git a/src/SlimVector.Storage/FileSystemDataGroupStorage.cs b/src/SlimVector.Storage/FileSystemDataGroupStorage.cs index 0731795..7645838 100644 --- a/src/SlimVector.Storage/FileSystemDataGroupStorage.cs +++ b/src/SlimVector.Storage/FileSystemDataGroupStorage.cs @@ -257,10 +257,36 @@ public long GetAllocatedBytes(string groupId) EnsureInitialized(); ValidateGroupId(groupId); string path = GetGroupPath(groupId); - return !Directory.Exists(path) - ? 0 - : Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories) - .Sum(static file => new FileInfo(file).Length); + if (!Directory.Exists(path)) + { + return 0; + } + + long allocatedBytes = 0; + try + { + foreach (string file in Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories)) + { + try + { + allocatedBytes = checked(allocatedBytes + new FileInfo(file).Length); + } + catch (FileNotFoundException) + { + // Atomic writes and compaction can move a file after it was enumerated. + } + catch (DirectoryNotFoundException) + { + // A nested directory can be moved while this point-in-time estimate is collected. + } + } + } + catch (DirectoryNotFoundException) + { + // The data group can be removed after the initial existence check. + } + + return allocatedBytes; } public void Dispose() diff --git a/tests/SlimVector.Raft.Tests/RaftClusterTests.cs b/tests/SlimVector.Raft.Tests/RaftClusterTests.cs index 2362448..9b28be0 100644 --- a/tests/SlimVector.Raft.Tests/RaftClusterTests.cs +++ b/tests/SlimVector.Raft.Tests/RaftClusterTests.cs @@ -168,6 +168,21 @@ await node.ReplicateDataAsync( Assert.All(groupIds, groupId => Assert.True(node.GetGroup(groupId).IsLeader)); } + [Fact] + public async Task MultiRaftDisposalIsIdempotent() + { + CancellationToken cancellationToken = TestContext.Current.CancellationToken; + cancellationToken.ThrowIfCancellationRequested(); + using TemporaryDirectory directory = new(); + IPEndPoint endpoint = AllocateLoopbackEndpoints(1)[0]; + MultiRaftNode node = new( + [Options(MultiRaftNode.CatalogGroupId, endpoint, [endpoint], directory.Path, 0)], + _ => new RecordingCommandApplier()); + + await node.DisposeAsync(); + await node.DisposeAsync(); + } + [Fact(Timeout = 45_000)] public async Task MultiRaftStartsAndRemovesADataGroupWhileRunning() { diff --git a/tests/SlimVector.Replication.Tests/GeoReplicationTests.cs b/tests/SlimVector.Replication.Tests/GeoReplicationTests.cs index 9c74cf8..c342a58 100644 --- a/tests/SlimVector.Replication.Tests/GeoReplicationTests.cs +++ b/tests/SlimVector.Replication.Tests/GeoReplicationTests.cs @@ -71,7 +71,10 @@ await WaitUntilAsync( await restarted.StartAsync(cancellationToken); CollectionDefinition updated = collection with { UpdatedAt = collection.UpdatedAt.AddMinutes(1) }; await restarted.PublishAsync(GeoReplicationEventFactory.CollectionUpsert(updated), cancellationToken); - await WaitUntilAsync(() => receiver.GetMetrics().ReceivedEvents == 3, cancellationToken); + await WaitUntilAsync( + () => receiver.GetMetrics().ReceivedEvents == 3 && + restarted.GetMetrics() is { PendingEvents: 0, SentEvents: 1 }, + cancellationToken); Assert.Equal(1, restarted.GetMetrics().SentEvents); } } diff --git a/tests/SlimVector.Storage.Tests/FileSystemStorageEngineTests.cs b/tests/SlimVector.Storage.Tests/FileSystemStorageEngineTests.cs index d779e5f..e8f029e 100644 --- a/tests/SlimVector.Storage.Tests/FileSystemStorageEngineTests.cs +++ b/tests/SlimVector.Storage.Tests/FileSystemStorageEngineTests.cs @@ -64,6 +64,50 @@ await groups.AppendAsync( Assert.True(File.Exists(Path.Combine(directory.Path, "cluster-topology-v2.json"))); } + [Fact] + public async Task DistributedStorageReportsAllocatedBytesDuringAtomicFileReplacement() + { + CancellationToken cancellationToken = TestContext.Current.CancellationToken; + using TemporaryDirectory directory = new(); + using FileSystemDataGroupStorage groups = new(new StorageSettings { Path = directory.Path }); + await groups.InitializeAsync(cancellationToken); + await groups.EnsureDataGroupAsync("data-0", cancellationToken); + + string groupPath = Path.Combine(directory.Path, "data-groups", "data-0"); + string target = Path.Combine(groupPath, "capacity-race.bin"); + byte[] contents = new byte[4_096]; + await File.WriteAllBytesAsync(target, contents, cancellationToken); + TaskCompletionSource start = new(TaskCreationOptions.RunContinuationsAsynchronously); + int observations = 0; + + Task writer = Task.Run( + async () => + { + await start.Task.WaitAsync(cancellationToken); + for (int iteration = 0; iteration < 2_000; iteration++) + { + string temporary = $"{target}.{iteration}.tmp"; + await File.WriteAllBytesAsync(temporary, contents, cancellationToken); + File.Move(temporary, target, overwrite: true); + } + }, + cancellationToken); + Task reader = Task.Run( + () => + { + start.SetResult(); + while (!writer.IsCompleted) + { + Assert.True(groups.GetAllocatedBytes("data-0") >= contents.Length); + observations++; + } + }, + cancellationToken); + + await Task.WhenAll(writer, reader); + Assert.True(observations > 0); + } + [Fact] public async Task LogicalIoMetricsAreMonotonicAndRecordDurableFlushes() {