Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ jobs:
run: dotnet build SlimVector.slnx -c Release --no-restore

- name: Run tests
run: dotnet test SlimVector.slnx -c Release --no-build --verbosity normal
run: dotnet test --solution SlimVector.slnx -c Release --no-build --verbosity normal

version:
name: Compute release channel and version
Expand Down
6 changes: 3 additions & 3 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,11 +72,11 @@ et les performances priment sur la commodité d’une modification locale.
Exemples :

```bash
dotnet test tests/SlimVector.Studio.Tests/SlimVector.Studio.Tests.csproj \
dotnet test --project tests/SlimVector.Studio.Tests/SlimVector.Studio.Tests.csproj \
--no-restore --filter "FullyQualifiedName~StudioIntegrationTests"

dotnet build SlimVector.slnx --no-restore
dotnet test SlimVector.slnx --no-restore --no-build
dotnet test --solution SlimVector.slnx --no-restore --no-build
```

## Benchmarks : toujours mesurer
Expand Down Expand Up @@ -148,7 +148,7 @@ Avant de conclure :
2. Les tests ciblés passent.
3. Le benchmark pertinent a été rejoué et ses résultats ont été examinés.
4. `dotnet build SlimVector.slnx --no-restore` passe sans avertissement.
5. `dotnet test SlimVector.slnx --no-restore --no-build` passe.
5. `dotnet test --solution SlimVector.slnx --no-restore --no-build` passe.
6. `git diff --check` ne signale aucune erreur.
7. Aucun serveur, processus, fichier temporaire ou artefact de test n’est laissé.
8. Le compte rendu cite les fichiers modifiés, les tests exécutés, les benchmarks
Expand Down
31 changes: 16 additions & 15 deletions Directory.Packages.props
Original file line number Diff line number Diff line change
Expand Up @@ -6,24 +6,25 @@
<ItemGroup>
<PackageVersion Include="BenchmarkDotNet" Version="0.15.8" />
<PackageVersion Include="DocumentFormat.OpenXml" Version="3.5.1" />
<PackageVersion Include="DotNext.Net.Cluster" Version="6.4.1" />
<PackageVersion Include="DotNext.Net.Cluster" Version="6.7.0" />
<PackageVersion Include="MemoryPack" Version="1.21.4" />
<PackageVersion Include="MessagePack" Version="3.1.8" />
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.10" />
<PackageVersion Include="Microsoft.AspNetCore.OpenApi" Version="10.0.10" />
<PackageVersion Include="Microsoft.Extensions.Http" Version="10.0.10" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.10" />
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.10" />
<PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.10" />
<PackageVersion Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="10.0.10" />
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.8.1" />
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.11" />
<PackageVersion Include="Microsoft.AspNetCore.OpenApi" Version="10.0.11" />
<PackageVersion Include="Microsoft.Extensions.Http" Version="10.0.11" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.11" />
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.11" />
<PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.11" />
<PackageVersion Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="10.0.11" />
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.9.0" />
<!-- Microsoft.AspNetCore.OpenApi 10.0.11 source generation still targets Microsoft.OpenApi 2.x. -->
<PackageVersion Include="Microsoft.OpenApi" Version="2.11.0" />
<PackageVersion Include="Microsoft.ML.OnnxRuntime" Version="1.27.1" />
<PackageVersion Include="PdfPig" Version="0.1.14" />
<PackageVersion Include="PdfPig.Rendering.Skia" Version="0.1.14.2" />
<PackageVersion Include="SkiaSharp" Version="3.119.4" />
<PackageVersion Include="Microsoft.ML.OnnxRuntime" Version="1.29.0" />
<PackageVersion Include="PdfPig" Version="0.1.16" />
<PackageVersion Include="PdfPig.Rendering.Skia" Version="0.1.16.2" />
<PackageVersion Include="SkiaSharp" Version="4.151.2" />
<PackageVersion Include="Tokenizers.HuggingFace" Version="3.23.1" />
<PackageVersion Include="xunit.v3" Version="3.2.2" />
<PackageVersion Include="xunit.runner.visualstudio" Version="3.1.5" />
<PackageVersion Include="xunit.v3" Version="4.0.0" />
<PackageVersion Include="xunit.runner.visualstudio" Version="4.0.0" />
</ItemGroup>
</Project>
5 changes: 3 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
# syntax=docker/dockerfile:1.7
FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build
FROM mcr.microsoft.com/dotnet/sdk:10.0.300 AS build
WORKDIR /src
RUN apt-get -o Acquire::Retries=5 update \
&& apt-get -o Acquire::Retries=5 install -y --no-install-recommends clang llvm zlib1g-dev \
&& rm -rf /var/lib/apt/lists/*
COPY . .
ARG TARGETARCH
RUN arch="$TARGETARCH"; \
RUN set -eu; \
arch="$TARGETARCH"; \
if [ "$arch" = "amd64" ]; then arch="x64"; fi; \
rid="linux-$arch"; \
dotnet restore src/SlimVector.Api/SlimVector.Api.csproj -r "$rid"; \
Expand Down
5 changes: 3 additions & 2 deletions Dockerfile.Studio
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
# syntax=docker/dockerfile:1.7
FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build
FROM mcr.microsoft.com/dotnet/sdk:10.0.300 AS build
WORKDIR /src
COPY . .
ARG TARGETARCH
RUN arch="$TARGETARCH"; \
RUN set -eu; \
arch="$TARGETARCH"; \
if [ "$arch" = "amd64" ]; then arch="x64"; fi; \
rid="linux-$arch"; \
dotnet restore src/SlimVector.Studio/SlimVector.Studio.csproj -r "$rid"; \
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ JSON is the default public wire format. For vector-heavy client traffic, constru
```bash
dotnet format SlimVector.slnx --verify-no-changes
dotnet build SlimVector.slnx -c Release
dotnet test SlimVector.slnx -c Release
dotnet test --solution SlimVector.slnx -c Release
dotnet publish src/SlimVector.Api/SlimVector.Api.csproj -c Release -r linux-x64 --self-contained true
```

Expand Down
2 changes: 1 addition & 1 deletion docs/testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ Run deterministic quality gates from the repository root:
```bash
dotnet format SlimVector.slnx --verify-no-changes
dotnet build SlimVector.slnx -c Release
dotnet test SlimVector.slnx -c Release
dotnet test --solution SlimVector.slnx -c Release
```

The test projects cover domain validation; SIMD Flat, HNSW, IVF-Flat, IVF-PQ, scalar quantization and SSD DiskANN persistence/recall/mutation/rollback; Auto selection, online migration, failed-candidate isolation and restart-safe rollback; BM25, metadata and hybrid fusion; immutable storage/crash reconciliation/compaction; CRUD, eviction, batching, backpressure and reserved/fair token buckets; backup/S3/restore; actual TCP Raft election/failover/snapshot/partition/multi-group plus fourth-member warm-up, restart and removal; geographic outage/restart/divergence; and HTTP/client/admin/429/redirect behavior.
Expand Down
3 changes: 3 additions & 0 deletions global.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,8 @@
"version": "10.0.300",
"rollForward": "latestPatch",
"allowPrerelease": false
},
"test": {
"runner": "Microsoft.Testing.Platform"
}
}
12 changes: 11 additions & 1 deletion src/SlimVector.Raft/MultiRaftNode.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ public sealed class MultiRaftNode : IAsyncDisposable
private readonly ConcurrentDictionary<string, RaftGroupNode> _groups = new(StringComparer.Ordinal);
private readonly Func<string, IRaftCommandApplier> _applierFactory;
private readonly SemaphoreSlim _lifecycleGate = new(1, 1);
private readonly object _disposeLock = new();
private Task? _disposeTask;
private volatile bool _started;

public MultiRaftNode(
Expand Down Expand Up @@ -226,7 +228,15 @@ public IReadOnlyList<ClusterMembershipStatus> GetMembershipStatuses() => _groups
.Select(static group => new ClusterMembershipStatus(group.GroupId, group.GetMemberStatuses(), null, null))
.ToArray();

public async ValueTask DisposeAsync()
public ValueTask DisposeAsync()
{
lock (_disposeLock)
{
return new ValueTask(_disposeTask ??= DisposeCoreAsync());
}
}

private async Task DisposeCoreAsync()
{
await _lifecycleGate.WaitAsync().ConfigureAwait(false);
try
Expand Down
34 changes: 30 additions & 4 deletions src/SlimVector.Storage/FileSystemDataGroupStorage.cs
Original file line number Diff line number Diff line change
Expand Up @@ -257,10 +257,36 @@ public long GetAllocatedBytes(string groupId)
EnsureInitialized();
ValidateGroupId(groupId);
string path = GetGroupPath(groupId);
return !Directory.Exists(path)
? 0
: Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories)
.Sum(static file => new FileInfo(file).Length);
if (!Directory.Exists(path))
{
return 0;
}

long allocatedBytes = 0;
try
{
foreach (string file in Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories))
{
try
{
allocatedBytes = checked(allocatedBytes + new FileInfo(file).Length);
}
catch (FileNotFoundException)
{
// Atomic writes and compaction can move a file after it was enumerated.
}
catch (DirectoryNotFoundException)
{
// A nested directory can be moved while this point-in-time estimate is collected.
}
}
}
catch (DirectoryNotFoundException)
{
// The data group can be removed after the initial existence check.
}

return allocatedBytes;
}

public void Dispose()
Expand Down
15 changes: 15 additions & 0 deletions tests/SlimVector.Raft.Tests/RaftClusterTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,21 @@ await node.ReplicateDataAsync(
Assert.All(groupIds, groupId => Assert.True(node.GetGroup(groupId).IsLeader));
}

[Fact]
public async Task MultiRaftDisposalIsIdempotent()
{
CancellationToken cancellationToken = TestContext.Current.CancellationToken;
cancellationToken.ThrowIfCancellationRequested();
using TemporaryDirectory directory = new();
IPEndPoint endpoint = AllocateLoopbackEndpoints(1)[0];
MultiRaftNode node = new(
[Options(MultiRaftNode.CatalogGroupId, endpoint, [endpoint], directory.Path, 0)],
_ => new RecordingCommandApplier());

await node.DisposeAsync();
await node.DisposeAsync();
}

[Fact(Timeout = 45_000)]
public async Task MultiRaftStartsAndRemovesADataGroupWhileRunning()
{
Expand Down
5 changes: 4 additions & 1 deletion tests/SlimVector.Replication.Tests/GeoReplicationTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,10 @@ await WaitUntilAsync(
await restarted.StartAsync(cancellationToken);
CollectionDefinition updated = collection with { UpdatedAt = collection.UpdatedAt.AddMinutes(1) };
await restarted.PublishAsync(GeoReplicationEventFactory.CollectionUpsert(updated), cancellationToken);
await WaitUntilAsync(() => receiver.GetMetrics().ReceivedEvents == 3, cancellationToken);
await WaitUntilAsync(
() => receiver.GetMetrics().ReceivedEvents == 3 &&
restarted.GetMetrics() is { PendingEvents: 0, SentEvents: 1 },
cancellationToken);
Assert.Equal(1, restarted.GetMetrics().SentEvents);
}
}
Expand Down
44 changes: 44 additions & 0 deletions tests/SlimVector.Storage.Tests/FileSystemStorageEngineTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,50 @@ await groups.AppendAsync(
Assert.True(File.Exists(Path.Combine(directory.Path, "cluster-topology-v2.json")));
}

[Fact]
public async Task DistributedStorageReportsAllocatedBytesDuringAtomicFileReplacement()
{
CancellationToken cancellationToken = TestContext.Current.CancellationToken;
using TemporaryDirectory directory = new();
using FileSystemDataGroupStorage groups = new(new StorageSettings { Path = directory.Path });
await groups.InitializeAsync(cancellationToken);
await groups.EnsureDataGroupAsync("data-0", cancellationToken);

string groupPath = Path.Combine(directory.Path, "data-groups", "data-0");
string target = Path.Combine(groupPath, "capacity-race.bin");
byte[] contents = new byte[4_096];
await File.WriteAllBytesAsync(target, contents, cancellationToken);
TaskCompletionSource start = new(TaskCreationOptions.RunContinuationsAsynchronously);
int observations = 0;

Task writer = Task.Run(
async () =>
{
await start.Task.WaitAsync(cancellationToken);
for (int iteration = 0; iteration < 2_000; iteration++)
{
string temporary = $"{target}.{iteration}.tmp";
await File.WriteAllBytesAsync(temporary, contents, cancellationToken);
File.Move(temporary, target, overwrite: true);
}
},
cancellationToken);
Task reader = Task.Run(
() =>
{
start.SetResult();
while (!writer.IsCompleted)
{
Assert.True(groups.GetAllocatedBytes("data-0") >= contents.Length);
observations++;
}
},
cancellationToken);

await Task.WhenAll(writer, reader);
Assert.True(observations > 0);
}

[Fact]
public async Task LogicalIoMetricsAreMonotonicAndRecordDurableFlushes()
{
Expand Down
Loading