From 5f3cd26d94e42b74f7ae125a73f3f01462f7ae8b Mon Sep 17 00:00:00 2001 From: Geovanny Alzate Sandoval Date: Wed, 8 Jul 2026 13:29:30 -0500 Subject: [PATCH 1/2] fix: serialize index writes per index to prevent Lucene corruption IndexDocuments creates a fresh IndexWriter per request. Lucene allows only one IndexWriter per directory, so two concurrent indexing requests against the same index race: the loser fails with LockObtainFailedException, and under sustained concurrency a torn commit corrupts the segment files permanently (FileNotFoundException: .../_N.si on every subsequent write until the index directory is deleted from disk). Reproduced with 30 concurrent writers x 30 sequential mergeOrUpload requests against a fresh index: 889/900 requests returned 500 and the index remained broken after concurrency stopped. With this change the same load succeeds 900/900 and the index stays healthy. A per-index SemaphoreSlim serializes IndexDocuments per index name; requests against different indexes still run in parallel. Co-Authored-By: Claude Fable 5 --- .../Indexing/LuceneNetSearchIndexer.cs | 70 ++++++++++++------- 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/AzureSearchEmulator/Indexing/LuceneNetSearchIndexer.cs b/AzureSearchEmulator/Indexing/LuceneNetSearchIndexer.cs index 24c515e..b1d0259 100644 --- a/AzureSearchEmulator/Indexing/LuceneNetSearchIndexer.cs +++ b/AzureSearchEmulator/Indexing/LuceneNetSearchIndexer.cs @@ -1,4 +1,5 @@ -using AzureSearchEmulator.Models; +using System.Collections.Concurrent; +using AzureSearchEmulator.Models; using AzureSearchEmulator.SearchData; using Lucene.Net.Index; using Lucene.Net.Util; @@ -10,41 +11,62 @@ public class LuceneNetSearchIndexer( ILuceneIndexReaderFactory luceneIndexReaderFactory) : ISearchIndexer { + // One writer gate per index. Lucene allows only a single IndexWriter per + // directory; this class creates a fresh writer per request, so two + // concurrent indexing requests against the same index would race — the + // loser either fails with LockObtainFailedException or, worse, observes + // a torn commit and permanently corrupts the segment files + // (FileNotFoundException: .../_N.si on every subsequent write until the + // index directory is deleted). Serializing IndexDocuments per index name + // removes both failure modes; requests against different indexes still + // run in parallel. + private static readonly ConcurrentDictionary IndexWriteGates = new(); + public IndexDocumentsResult IndexDocuments(SearchIndex index, IList actions) { - var analyzer = AnalyzerHelper.GetPerFieldIndexAnalyzer(index.Fields); + var gate = IndexWriteGates.GetOrAdd(index.Name, static _ => new SemaphoreSlim(1, 1)); + gate.Wait(); - var config = new IndexWriterConfig(LuceneVersion.LUCENE_48, analyzer); + try + { + var analyzer = AnalyzerHelper.GetPerFieldIndexAnalyzer(index.Fields); - var directory = luceneDirectoryFactory.GetDirectory(index.Name); - using var writer = new IndexWriter(directory, config); + var config = new IndexWriterConfig(LuceneVersion.LUCENE_48, analyzer); - var key = index.GetKeyField(); + var directory = luceneDirectoryFactory.GetDirectory(index.Name); + using var writer = new IndexWriter(directory, config); - var results = new IndexDocumentsResult(); + var key = index.GetKeyField(); - // ReSharper disable once AccessToDisposedClosure - var readerLazy = new Lazy(() => writer.GetReader(true)); + var results = new IndexDocumentsResult(); - var context = new IndexingContext(index, key, writer, readerLazy); + // ReSharper disable once AccessToDisposedClosure + var readerLazy = new Lazy(() => writer.GetReader(true)); - foreach (var action in actions) - { - var result = action.PerformIndexingAsync(context); - results.Value.Add(result); - } + var context = new IndexingContext(index, key, writer, readerLazy); - if (readerLazy.IsValueCreated) - { - var reader = readerLazy.Value; - reader.Dispose(); - } + foreach (var action in actions) + { + var result = action.PerformIndexingAsync(context); + results.Value.Add(result); + } + + if (readerLazy.IsValueCreated) + { + var reader = readerLazy.Value; + reader.Dispose(); + } - writer.Commit(); - writer.Flush(true, true); + writer.Commit(); + writer.Flush(true, true); - luceneIndexReaderFactory.RefreshReader(index.Name); + luceneIndexReaderFactory.RefreshReader(index.Name); - return results; + return results; + } + finally + { + gate.Release(); + } } } From 98b5416c25ea7dfbe462912558d93aa6461794c5 Mon Sep 17 00:00:00 2001 From: Geovanny Alzate Sandoval Date: Wed, 22 Jul 2026 14:25:01 -0500 Subject: [PATCH 2/2] =?UTF-8?q?[stream:ops]=20feat:=20request/indexing=20o?= =?UTF-8?q?bservability=20=E2=80=94=20status=20codes,=20batch=20bodies,=20?= =?UTF-8?q?per-item=20outcomes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The emulator logged only [HTTP METHOD] path — no response status, no batch body, no per-item results. A rejected or dropped index merge was therefore invisible: the Azure SDK IndexDocuments does not throw on per-item failures by default, so neither the client nor the emulator left any trace (cost a full diagnosis session on the Curbit devbox: an order READIED merge reached the emulator but the doc never showed READIED, and nothing said why). - request middleware logs the response status after the pipeline ([HTTP POST 207] ...) and any unhandled exception with type + message - search.index logs the full batch body before deserialization (a malformed or wrong-fields payload is otherwise undiagnosable) - search.index logs per-item outcomes keyed ([INDEX ] 5/6 ok — key:200, key:404 FAILED(...)) — the only place a dropped write is guaranteed to surface Co-Authored-By: Claude Opus 4.8 --- .../Controllers/DocumentIndexingController.cs | 14 ++++++++++++++ AzureSearchEmulator/Program.cs | 17 ++++++++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/AzureSearchEmulator/Controllers/DocumentIndexingController.cs b/AzureSearchEmulator/Controllers/DocumentIndexingController.cs index b5f4258..bb7fb88 100644 --- a/AzureSearchEmulator/Controllers/DocumentIndexingController.cs +++ b/AzureSearchEmulator/Controllers/DocumentIndexingController.cs @@ -30,6 +30,12 @@ public async Task IndexDocuments(string indexKey) using var sr = new StreamReader(Request.Body); var json = await sr.ReadToEndAsync(); + + // Full batch body — logged BEFORE deserialization so even a malformed + // payload is visible. Devbox visibility beats log volume here: a merge + // whose body carried the wrong fields is otherwise undiagnosable. + Console.WriteLine($"[INDEX {indexKey}] body: {json}"); + var batch = JsonSerializer.Deserialize(json, jsonSerializerOptions); if (batch == null) @@ -65,6 +71,14 @@ public async Task IndexDocuments(string indexKey) var result = searchIndexer.IndexDocuments(index, actions); + // Per-item outcomes, keyed — the Azure SDK's IndexDocuments does NOT + // throw on per-item failures by default, so a rejected merge is + // invisible to the caller unless it checks the batch response. This + // log line is the only place a dropped write is guaranteed to surface. + var summary = string.Join(", ", result.Value.Select(i => + i.Status ? $"{i.Key}:{i.StatusCode}" : $"{i.Key}:{i.StatusCode} FAILED({i.ErrorMessage})")); + Console.WriteLine($"[INDEX {indexKey}] {result.Value.Count(i => i.Status)}/{actions.Count} ok — {summary}"); + return StatusCode(result.Value.Any(i => !i.Status) ? 207 : 200, result); } } diff --git a/AzureSearchEmulator/Program.cs b/AzureSearchEmulator/Program.cs index 61a2167..237eb46 100644 --- a/AzureSearchEmulator/Program.cs +++ b/AzureSearchEmulator/Program.cs @@ -73,14 +73,25 @@ app.UseRouting(); -app.Use((context, next) => +app.Use(async (context, next) => { var method = context.Request.Method; var path = context.Request.Path; var queryString = context.Request.QueryString.ToString(); var fullPath = string.IsNullOrEmpty(queryString) ? path.ToString() : $"{path}{queryString}"; - Console.WriteLine($"[HTTP {method}] {fullPath}"); - return next(); + try + { + await next(); + // Status code logged AFTER the pipeline so a rejected request is + // visible in the log — a silent 4xx/5xx here cost a full afternoon + // of diagnosis when an index merge was dropped without a trace. + Console.WriteLine($"[HTTP {method} {context.Response.StatusCode}] {fullPath}"); + } + catch (Exception ex) + { + Console.WriteLine($"[HTTP {method} EXCEPTION] {fullPath} — {ex.GetType().Name}: {ex.Message}"); + throw; + } }); app.MapControllers();