Fix permanent Lucene index corruption under concurrent indexing requests

AzureSearchEmulator/Controllers/DocumentIndexingController.cs

@@ -30,6 +30,12 @@ public async Task<IActionResult> IndexDocuments(string indexKey)
 
         using var sr = new StreamReader(Request.Body);
         var json = await sr.ReadToEndAsync();
+
+        // Full batch body — logged BEFORE deserialization so even a malformed
+        // payload is visible. Devbox visibility beats log volume here: a merge
+        // whose body carried the wrong fields is otherwise undiagnosable.
+        Console.WriteLine($"[INDEX {indexKey}] body: {json}");
+
         var batch = JsonSerializer.Deserialize<IndexDocumentsBatch>(json, jsonSerializerOptions);
 
         if (batch == null)
@@ -65,6 +71,14 @@ public async Task<IActionResult> IndexDocuments(string indexKey)
 
         var result = searchIndexer.IndexDocuments(index, actions);
 
+        // Per-item outcomes, keyed — the Azure SDK's IndexDocuments does NOT
+        // throw on per-item failures by default, so a rejected merge is
+        // invisible to the caller unless it checks the batch response. This
+        // log line is the only place a dropped write is guaranteed to surface.
+        var summary = string.Join(", ", result.Value.Select(i =>
+            i.Status ? $"{i.Key}:{i.StatusCode}" : $"{i.Key}:{i.StatusCode} FAILED({i.ErrorMessage})"));
+        Console.WriteLine($"[INDEX {indexKey}] {result.Value.Count(i => i.Status)}/{actions.Count} ok — {summary}");
+
         return StatusCode(result.Value.Any(i => !i.Status) ? 207 : 200, result);
     }
 }

AzureSearchEmulator/Indexing/LuceneNetSearchIndexer.cs

@@ -1,4 +1,5 @@
-using AzureSearchEmulator.Models;
+using System.Collections.Concurrent;
+using AzureSearchEmulator.Models;
 using AzureSearchEmulator.SearchData;
 using Lucene.Net.Index;
 using Lucene.Net.Util;
@@ -10,41 +11,62 @@ public class LuceneNetSearchIndexer(
     ILuceneIndexReaderFactory luceneIndexReaderFactory)
     : ISearchIndexer
 {
+    // One writer gate per index. Lucene allows only a single IndexWriter per
+    // directory; this class creates a fresh writer per request, so two
+    // concurrent indexing requests against the same index would race — the
+    // loser either fails with LockObtainFailedException or, worse, observes
+    // a torn commit and permanently corrupts the segment files
+    // (FileNotFoundException: .../_N.si on every subsequent write until the
+    // index directory is deleted). Serializing IndexDocuments per index name
+    // removes both failure modes; requests against different indexes still
+    // run in parallel.
+    private static readonly ConcurrentDictionary<string, SemaphoreSlim> IndexWriteGates = new();
+
     public IndexDocumentsResult IndexDocuments(SearchIndex index, IList<IndexDocumentAction> actions)
     {
-        var analyzer = AnalyzerHelper.GetPerFieldIndexAnalyzer(index.Fields);
+        var gate = IndexWriteGates.GetOrAdd(index.Name, static _ => new SemaphoreSlim(1, 1));
+        gate.Wait();
 
-        var config = new IndexWriterConfig(LuceneVersion.LUCENE_48, analyzer);
+        try
+        {
+            var analyzer = AnalyzerHelper.GetPerFieldIndexAnalyzer(index.Fields);
 
-        var directory = luceneDirectoryFactory.GetDirectory(index.Name);
-        using var writer = new IndexWriter(directory, config);
+            var config = new IndexWriterConfig(LuceneVersion.LUCENE_48, analyzer);
 
-        var key = index.GetKeyField();
+            var directory = luceneDirectoryFactory.GetDirectory(index.Name);
+            using var writer = new IndexWriter(directory, config);
 
-        var results = new IndexDocumentsResult();
+            var key = index.GetKeyField();
 
-        // ReSharper disable once AccessToDisposedClosure
-        var readerLazy = new Lazy<IndexReader>(() => writer.GetReader(true));
+            var results = new IndexDocumentsResult();
 
-        var context = new IndexingContext(index, key, writer, readerLazy);
+            // ReSharper disable once AccessToDisposedClosure
+            var readerLazy = new Lazy<IndexReader>(() => writer.GetReader(true));
 
-        foreach (var action in actions)
-        {
-            var result = action.PerformIndexingAsync(context);
-            results.Value.Add(result);
-        }
+            var context = new IndexingContext(index, key, writer, readerLazy);
 
-        if (readerLazy.IsValueCreated)
-        {
-            var reader = readerLazy.Value;
-            reader.Dispose();
-        }
+            foreach (var action in actions)
+            {
+                var result = action.PerformIndexingAsync(context);
+                results.Value.Add(result);
+            }
+
+            if (readerLazy.IsValueCreated)
+            {
+                var reader = readerLazy.Value;
+                reader.Dispose();
+            }
 
-        writer.Commit();
-        writer.Flush(true, true);
+            writer.Commit();
+            writer.Flush(true, true);
 
-        luceneIndexReaderFactory.RefreshReader(index.Name);
+            luceneIndexReaderFactory.RefreshReader(index.Name);
 
-        return results;
+            return results;
+        }
+        finally
+        {
+            gate.Release();
+        }
     }
 }

AzureSearchEmulator/Program.cs

@@ -73,14 +73,25 @@
 
 app.UseRouting();
 
-app.Use((context, next) =>
+app.Use(async (context, next) =>
 {
     var method = context.Request.Method;
     var path = context.Request.Path;
     var queryString = context.Request.QueryString.ToString();
     var fullPath = string.IsNullOrEmpty(queryString) ? path.ToString() : $"{path}{queryString}";
-    Console.WriteLine($"[HTTP {method}] {fullPath}");
-    return next();
+    try
+    {
+        await next();
+        // Status code logged AFTER the pipeline so a rejected request is
+        // visible in the log — a silent 4xx/5xx here cost a full afternoon
+        // of diagnosis when an index merge was dropped without a trace.
+        Console.WriteLine($"[HTTP {method} {context.Response.StatusCode}] {fullPath}");
+    }
+    catch (Exception ex)
+    {
+        Console.WriteLine($"[HTTP {method} EXCEPTION] {fullPath} — {ex.GetType().Name}: {ex.Message}");
+        throw;
+    }
 });
 
 app.MapControllers();