Ingestion re-implement on updated Elastic.Ingest.Elasticsearch

Directory.Packages.props

@@ -48,7 +48,8 @@
     <PackageVersion Include="Elastic.Aspire.Hosting.Elasticsearch" Version="9.3.0" />
     <PackageVersion Include="Elastic.Clients.Elasticsearch" Version="9.3.0" />
     <PackageVersion Include="FakeItEasy" Version="9.0.1" />
-    <PackageVersion Include="Elastic.Ingest.Elasticsearch" Version="0.17.1" />
+    <PackageVersion Include="Elastic.Ingest.Elasticsearch" Version="0.24.0" />
+    <PackageVersion Include="Elastic.Mapping" Version="0.24.0" />
     <PackageVersion Include="InMemoryLogger" Version="1.0.66" />
     <PackageVersion Include="MartinCostello.Logging.XUnit.v3" Version="0.7.0" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.0" />

docs/cli/assembler/assembler-index.md

@@ -29,9 +29,6 @@ docs-builder assembler index [options...] [-h|--help] [--version]
 `--password` `<string>`
 :   Elasticsearch password (basic auth), alternatively set env DOCUMENTATION_ELASTIC_PASSWORD (optional)
 
-`--no-semantic` `<bool?>`
-:   Index without semantic fields (optional)
-
 `--search-num-threads` `<int?>`
 :   The number of search threads the inference endpoint should use. Defaults:   8 (optional)
 

docs/cli/docset/index-command.md

@@ -25,9 +25,6 @@ docs-builder index [options...] [-h|--help] [--version]
 `--password` `<string>`
 :   Elasticsearch password (basic auth), alternatively set env DOCUMENTATION_ELASTIC_PASSWORD (optional)
 
-`--no-semantic` `<bool?>`
-:   Index without semantic fields (optional)
-
 `--search-num-threads` `<int?>`
 :   The number of search threads the inference endpoint should use. Defaults:   8 (optional)
 

src/Elastic.Documentation.Configuration/DocumentationEndpoints.cs

@@ -9,6 +9,7 @@ namespace Elastic.Documentation.Configuration;
 public class DocumentationEndpoints
 {
 	public required ElasticsearchEndpoint Elasticsearch { get; init; }
+	public string Namespace { get; set; } = "dev";
 }
 
 public class ElasticsearchEndpoint
@@ -43,7 +44,6 @@ public class ElasticsearchEndpoint
 	public X509Certificate? Certificate { get; set; }
 	public bool CertificateIsNotRoot { get; set; }
 	public int? BootstrapTimeout { get; set; }
-	public bool NoSemantic { get; set; }
 	public bool ForceReindex { get; set; }
 
 	/// <summary>

src/Elastic.Documentation.Configuration/ElasticsearchEndpointConfigurator.cs

@@ -21,7 +21,6 @@ public record ElasticsearchIndexOptions
 	public string? Password { get; init; }
 
 	// inference options
-	public bool? NoSemantic { get; init; }
 	public bool? EnableAiEnrichment { get; init; }
 	public int? SearchNumThreads { get; init; }
 	public int? IndexNumThreads { get; init; }
@@ -117,8 +116,6 @@ public static async Task ApplyAsync(
 		if (options.BootstrapTimeout.HasValue)
 			cfg.BootstrapTimeout = options.BootstrapTimeout.Value;
 
-		if (options.NoSemantic.HasValue)
-			cfg.NoSemantic = options.NoSemantic.Value;
 		if (options.EnableAiEnrichment.HasValue)
 			cfg.EnableAiEnrichment = options.EnableAiEnrichment.Value;
 		if (options.ForceReindex.HasValue)

src/Elastic.Documentation.ServiceDefaults/AppDefaultsExtensions.cs

@@ -45,6 +45,9 @@ public static TBuilder AddDocumentationServiceDefaults<TBuilder>(this TBuilder b
 		_ = builder.Services.AddElasticDocumentationLogging(globalArgs.LogLevel, noConsole: globalArgs.IsMcp);
 		_ = services.AddSingleton(globalArgs);
 
+		var endpoints = ElasticsearchEndpointFactory.Create(builder.Configuration);
+		_ = services.AddSingleton(endpoints);
+
 		return builder.AddServiceDefaults();
 	}
 

src/Elastic.Documentation.ServiceDefaults/Elastic.Documentation.ServiceDefaults.csproj

@@ -17,6 +17,7 @@
     <PackageReference Include="OpenTelemetry.Instrumentation.Http"/>
     <PackageReference Include="OpenTelemetry.Instrumentation.Runtime"/>
     <PackageReference Include="Crayon"/>
+    <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets"/>
     <PackageReference Include="GitHub.Actions.Core" />
   </ItemGroup>
 

src/Elastic.Documentation.ServiceDefaults/ElasticsearchEndpointFactory.cs

@@ -0,0 +1,94 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Elastic.Documentation.Configuration;
+using Microsoft.Extensions.Configuration;
+
+namespace Elastic.Documentation.ServiceDefaults;
+
+/// <summary>Centralizes user-secrets + env-var reading for Elasticsearch configuration.</summary>
+public static class ElasticsearchEndpointFactory
+{
+	private const string UserSecretsId = "72f50f33-6fb9-4d08-bff3-39568fe370b3";
+
+	/// <summary>
+	/// Creates <see cref="DocumentationEndpoints"/> from user secrets and environment variables.
+	/// Returns <c>null</c> when no URL is available.
+	/// </summary>
+	public static DocumentationEndpoints Create(IConfiguration? appConfiguration = null)
+	{
+		var configBuilder = new ConfigurationBuilder();
+		_ = configBuilder.AddUserSecrets(UserSecretsId);
+		_ = configBuilder.AddEnvironmentVariables();
+		var config = configBuilder.Build();
+
+		var url =
+			config["Parameters:DocumentationElasticUrl"]
+			?? config["DOCUMENTATION_ELASTIC_URL"];
+
+		var apiKey =
+			config["Parameters:DocumentationElasticApiKey"]
+			?? config["DOCUMENTATION_ELASTIC_APIKEY"];
+
+		var password =
+			config["Parameters:DocumentationElasticPassword"]
+			?? config["DOCUMENTATION_ELASTIC_PASSWORD"];
+
+		var username =
+			config["Parameters:DocumentationElasticUsername"]
+			?? config["DOCUMENTATION_ELASTIC_USERNAME"]
+			?? "elastic";
+
+		if (string.IsNullOrEmpty(url))
+		{
+			return new DocumentationEndpoints
+			{
+				Elasticsearch = new ElasticsearchEndpoint { Uri = new Uri("http://localhost:9200") }
+			};
+		}
+
+		var endpoint = new ElasticsearchEndpoint
+		{
+			Uri = new Uri(url),
+			ApiKey = apiKey,
+			Password = password,
+			Username = username
+		};
+
+		var ns = ResolveNamespace(config, appConfiguration, endpoint.IndexNamePrefix);
+
+		return new DocumentationEndpoints { Elasticsearch = endpoint, Namespace = ns };
+	}
+
+	/// <summary>
+	/// Resolves the deployment namespace using this priority:
+	/// 1. <c>DOCUMENTATION_ELASTIC_INDEX</c> env var — strip prefix and <c>-latest</c> suffix
+	/// 2. <c>DOTNET_ENVIRONMENT</c> env var
+	/// 3. <c>ENVIRONMENT</c> env var
+	/// 4. Fallback: <c>"dev"</c>
+	/// </summary>
+	private static string ResolveNamespace(IConfiguration config, IConfiguration? appConfiguration, string indexNamePrefix)
+	{
+		var indexName = appConfiguration?["DOCUMENTATION_ELASTIC_INDEX"]
+			?? config["DOCUMENTATION_ELASTIC_INDEX"];
+
+		if (!string.IsNullOrEmpty(indexName))
+		{
+			var prefix = $"{indexNamePrefix}-";
+			const string suffix = "-latest";
+			if (indexName.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) &&
+				indexName.EndsWith(suffix, StringComparison.OrdinalIgnoreCase))
+			{
+				var ns = indexName[prefix.Length..^suffix.Length];
+				if (!string.IsNullOrEmpty(ns))
+					return ns;
+			}
+		}
+
+		var env = config["DOTNET_ENVIRONMENT"]
+			?? config["ENVIRONMENT"];
+
+		return !string.IsNullOrEmpty(env) ? env.ToLowerInvariant() : "dev";
+	}
+}

src/Elastic.Documentation/Elastic.Documentation.csproj

@@ -9,6 +9,7 @@
 
   <ItemGroup>
     <PackageReference Include="DotNet.Glob" />
+    <PackageReference Include="Elastic.Mapping" />
     <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" />
     <PackageReference Include="Microsoft.Extensions.Logging"/>
     <PackageReference Include="NetEscapades.EnumGenerators"/>

src/Elastic.Documentation/Search/ContentHash.cs

@@ -0,0 +1,19 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Security.Cryptography;
+using System.Text;
+
+namespace Elastic.Documentation.Search;
+
+/// <summary>Creates a short hex hash from one or more string components.</summary>
+public static class ContentHash
+{
+	/// <summary>
+	/// Concatenates all components, computes SHA-256, and returns the first 16 hex characters (lowercased).
+	/// Compatible with <c>HashedBulkUpdate.CreateHash</c>.
+	/// </summary>
+	public static string Create(params string[] components) =>
+		Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(string.Join("", components))))[..16].ToLowerInvariant();
+}

src/Elastic.Documentation/Search/DocumentationDocument.cs

@@ -4,6 +4,7 @@
 
 using System.Text.Json.Serialization;
 using Elastic.Documentation.AppliesTo;
+using Elastic.Mapping;
 
 namespace Elastic.Documentation.Search;
 
@@ -12,6 +13,7 @@ public record ParentDocument
 	[JsonPropertyName("title")]
 	public required string Title { get; set; }
 
+	[Keyword]
 	[JsonPropertyName("url")]
 	public required string Url { get; set; }
 }
@@ -28,27 +30,34 @@ public record DocumentationDocument
 	[JsonPropertyName("search_title")]
 	public required string SearchTitle { get; set; }
 
+	[Keyword(Normalizer = "keyword_normalizer")]
 	[JsonPropertyName("type")]
 	public required string Type { get; set; } = "doc";
 
 	/// <summary>
 	/// The canonical/primary product for this document (nested object with id and repository).
 	/// Name and version are looked up dynamically by product id.
 	/// </summary>
+	[Object]
 	[JsonPropertyName("product")]
 	[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
 	public IndexedProduct? Product { get; set; }
 
 	/// <summary>
 	/// All related products found during inference (from legacy mappings, applicability, etc.)
 	/// </summary>
+	[Object]
 	[JsonPropertyName("related_products")]
 	[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
 	public IndexedProduct[]? RelatedProducts { get; set; }
 
+	[Id]
+	[Keyword]
 	[JsonPropertyName("url")]
 	public required string Url { get; set; } = string.Empty;
 
+	[ContentHash]
+	[Keyword]
 	[JsonPropertyName("hash")]
 	public string Hash { get; set; } = string.Empty;
 
@@ -58,6 +67,7 @@ public record DocumentationDocument
 	[JsonPropertyName("navigation_table_of_contents")]
 	public int NavigationTableOfContents { get; set; } = 50; //default to a high number so that omission gets penalized.
 
+	[Keyword(Normalizer = "keyword_normalizer")]
 	[JsonPropertyName("navigation_section")]
 	public string? NavigationSection { get; set; }
 
@@ -67,18 +77,21 @@ public record DocumentationDocument
 	public DateTimeOffset BatchIndexDate { get; set; }
 
 	/// The date this document was last updated,
+	[Timestamp]
 	[JsonPropertyName("last_updated")]
 	public DateTimeOffset LastUpdated { get; set; }
 
 	[JsonPropertyName("description")]
 	public string? Description { get; set; }
 
+	[Text]
 	[JsonPropertyName("headings")]
 	public string[] Headings { get; set; } = [];
 
 	[JsonPropertyName("links")]
 	public string[] Links { get; set; } = [];
 
+	[Nested]
 	[JsonPropertyName("applies_to")]
 	public ApplicableTo? Applies { get; set; }
 
@@ -92,6 +105,7 @@ public record DocumentationDocument
 	[JsonPropertyName("abstract")]
 	public string? Abstract { get; set; }
 
+	[Object]
 	[JsonPropertyName("parents")]
 	public ParentDocument[] Parents { get; set; } = [];
 
@@ -105,41 +119,47 @@ public record DocumentationDocument
 	/// Key for enrichment cache lookups. Derived from normalized content + prompt hash.
 	/// Used by enrich processor to join AI-generated fields at index time.
 	/// </summary>
+	[Keyword]
 	[JsonPropertyName("enrichment_key")]
 	[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
 	public string? EnrichmentKey { get; set; }
 
 	/// <summary>
 	/// 3-5 sentences dense with technical entities, API names, and core functionality for vector matching.
 	/// </summary>
+	[Text]
 	[JsonPropertyName("ai_rag_optimized_summary")]
 	[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
 	public string? AiRagOptimizedSummary { get; set; }
 
 	/// <summary>
 	/// Exactly 5-10 words for a UI tooltip.
 	/// </summary>
+	[Text]
 	[JsonPropertyName("ai_short_summary")]
 	[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
 	public string? AiShortSummary { get; set; }
 
 	/// <summary>
 	/// A 3-8 word keyword string representing a high-intent user search for this doc.
 	/// </summary>
+	[Keyword]
 	[JsonPropertyName("ai_search_query")]
 	[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
 	public string? AiSearchQuery { get; set; }
 
 	/// <summary>
 	/// Array of 3-5 specific questions answered by this document.
 	/// </summary>
+	[Text]
 	[JsonPropertyName("ai_questions")]
 	[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
 	public string[]? AiQuestions { get; set; }
 
 	/// <summary>
 	/// Array of 2-4 specific use cases this doc helps with.
 	/// </summary>
+	[Text]
 	[JsonPropertyName("ai_use_cases")]
 	[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
 	public string[]? AiUseCases { get; set; }
@@ -148,6 +168,7 @@ public record DocumentationDocument
 	/// Hash of the LLM prompt templates used to generate AI fields.
 	/// Used to detect stale enrichments when prompts change.
 	/// </summary>
+	[Keyword]
 	[JsonPropertyName("enrichment_prompt_hash")]
 	[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
 	public string? EnrichmentPromptHash { get; set; }