From fefa1e8f036eabcad73a599ea378b3b6e3dde259 Mon Sep 17 00:00:00 2001 From: Serhiy Bzhezytskyy Date: Wed, 19 Aug 2026 16:56:30 +0300 Subject: [PATCH] SOLR-18376: remove ScalarQuantizedDenseVectorField compress and confidenceInterval The two fields and their package-private, @VisibleForTesting accessors go - 0 callers outside three tests. The schema params (compress, confidenceInterval, dynamicConfidenceInterval) stay consumed from args and only warned about: FieldType.setArgs throws on anything init() leaves behind, so removing the consumption would break every deployed schema still setting them - same shape as this file's own compressThreshold precedent at FieldType.java:187. The 4-argument constructor drops the two parameters. It had zero callers anywhere in the tree and now mirrors DenseVectorField's own 3-argument form; called out in the changelog as source-incompatible. None of the three affected tests were deleted - the two that asserted only the removed values now assert what survives and pin that those schemas still load, which nothing named before. Verified: compileJava/compileTestJava, ecjLintMain/Test, spotlessJavaCheck, renderJavadoc, zero external callers of the removed accessors, and ScalarQuantizedDenseVectorFieldTest - 7 tests, 0 failures. AI-assisted (Claude Sonnet 5) --- ...rquantized-compress-confidenceinterval.yml | 8 +++ .../ScalarQuantizedDenseVectorField.java | 55 ++----------------- .../ScalarQuantizedDenseVectorFieldTest.java | 19 +++---- 3 files changed, 22 insertions(+), 60 deletions(-) create mode 100644 changelog/unreleased/SOLR-18376-remove-scalarquantized-compress-confidenceinterval.yml diff --git a/changelog/unreleased/SOLR-18376-remove-scalarquantized-compress-confidenceinterval.yml b/changelog/unreleased/SOLR-18376-remove-scalarquantized-compress-confidenceinterval.yml new file mode 100644 index 000000000000..ffd10e67b139 --- /dev/null +++ b/changelog/unreleased/SOLR-18376-remove-scalarquantized-compress-confidenceinterval.yml @@ -0,0 +1,8 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: Remove the deprecated ScalarQuantizedDenseVectorField compress and confidenceInterval fields and their accessors useCompression and getConfidenceInterval, since Solr 10.1 no longer used by the underlying Lucene codec; the compress, confidenceInterval and dynamicConfidenceInterval schema params are still accepted and ignored with a deprecation warning, so an existing schema that sets them keeps loading. The public four-argument constructor loses its two corresponding parameters and now takes dimension, similarityFunction, vectorEncoding and bits. +type: removed +authors: + - name: Serhiy Bzhezytskyy +links: + - name: SOLR-18376 + url: https://issues.apache.org/jira/browse/SOLR-18376 diff --git a/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java b/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java index 26a33be48a24..66f0387505d1 100644 --- a/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java +++ b/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java @@ -18,7 +18,6 @@ import static java.util.Optional.ofNullable; -import com.google.common.annotations.VisibleForTesting; import java.util.Map; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat; @@ -36,7 +35,6 @@ public class ScalarQuantizedDenseVectorField extends DenseVectorField { "compress"; // can only be enabled when bits = 4 per Lucene codec spec static final int DEFAULT_BITS = 7; // use signed byte as default when unspecified - static final Float DEFAULT_CONFIDENCE_INTERVAL = null; // use dimension scaled confidence interval /** * Number of bits to use for storage Must be 4 (half-byte) or 7 (signed-byte) per Lucene codec @@ -44,24 +42,6 @@ public class ScalarQuantizedDenseVectorField extends DenseVectorField { */ private int bits; - /** - * Confidence interval to use for scalar quantization Default is calculated as - * `1-1/(vector_dimensions + 1)` - * - * @deprecated Since Solr 10.1. No longer used by the underlying Lucene codec. - */ - @Deprecated(since = "10.1") - private Float confidenceInterval; - - /** - * When enabled, in conjunction with 4 bit size, will pair values into single bytes for 50% - * reduction in memory usage (comes at the cost of some decode speed penalty) - * - * @deprecated Since Solr 10.1. No longer used by the underlying Lucene codec. - */ - @Deprecated(since = "10.1") - private boolean compress; - public ScalarQuantizedDenseVectorField() { super(); } @@ -70,13 +50,9 @@ public ScalarQuantizedDenseVectorField( int dimension, VectorSimilarityFunction similarityFunction, VectorEncoding vectorEncoding, - int bits, - Float confidenceInterval, - boolean compress) { + int bits) { super(dimension, similarityFunction, vectorEncoding); this.bits = bits; - this.confidenceInterval = confidenceInterval; - this.compress = compress; } @Override @@ -84,11 +60,12 @@ public void init(IndexSchema schema, Map args) { this.bits = ofNullable(args.remove(BITS_PARAM)).map(Integer::parseInt).orElse(DEFAULT_BITS); // These params ("compress", "confidenceInterval", "dynamicConfidenceInterval") are deprecated - // since Solr 10.1. Lucene 10.4's scalar-quantized vector format no longer consumes them - // directly. They are parsed for backward compatibility but are no-ops going forward. + // since Solr 10.1: Lucene's scalar-quantized vector format no longer consumes them, and as of + // 11.0 nothing here retains their values either. They are still consumed from the args - and + // only warned about - so that an existing schema setting them keeps loading instead of failing + // FieldType.setArgs' "invalid arguments" check. String compressStr = args.remove(COMPRESS_PARAM); if (compressStr != null) { - this.compress = Boolean.parseBoolean(compressStr); DeprecationLog.log( COMPRESS_PARAM, "The '" @@ -99,20 +76,16 @@ public void init(IndexSchema schema, Map args) { String confidenceIntervalStr = args.remove(CONFIDENCE_INTERVAL_PARAM); if (confidenceIntervalStr != null) { - this.confidenceInterval = Float.parseFloat(confidenceIntervalStr); DeprecationLog.log( CONFIDENCE_INTERVAL_PARAM, "The '" + CONFIDENCE_INTERVAL_PARAM + "' parameter for ScalarQuantizedDenseVectorField is deprecated since Solr 10.1" + " and will be ignored. Please remove it from your schema."); - } else { - this.confidenceInterval = DEFAULT_CONFIDENCE_INTERVAL; } String dynamicConfidenceIntervalStr = args.remove(DYNAMIC_CONFIDENCE_INTERVAL_PARAM); if (Boolean.parseBoolean(dynamicConfidenceIntervalStr)) { - this.confidenceInterval = 0f; DeprecationLog.log( DYNAMIC_CONFIDENCE_INTERVAL_PARAM, "The '" @@ -156,22 +129,4 @@ public void checkSchemaField(final SchemaField field) throws SolrException { public int getBits() { return bits; } - - /** - * @deprecated Since Solr 10.1. No longer used by the underlying Lucene codec. - */ - @Deprecated(since = "10.1") - @VisibleForTesting - boolean useCompression() { - return compress; - } - - /** - * @deprecated Since Solr 10.1. No longer used by the underlying Lucene codec. - */ - @Deprecated(since = "10.1") - @VisibleForTesting - Float getConfidenceInterval() { - return confidenceInterval; - } } diff --git a/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java b/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java index 94ebc63003be..cb1f24b88d5c 100644 --- a/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java +++ b/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java @@ -47,10 +47,6 @@ public void fieldDefinition_default_shouldLoadSchemaField() throws Exception { assertThat(defaultVectorType.getDimension(), is(4)); assertThat(defaultVectorType.getKnnAlgorithm(), is("hnsw")); assertThat(defaultVectorType.getBits(), is(ScalarQuantizedDenseVectorField.DEFAULT_BITS)); - assertThat( - defaultVectorType.getConfidenceInterval(), - is(ScalarQuantizedDenseVectorField.DEFAULT_CONFIDENCE_INTERVAL)); - assertThat(defaultVectorType.useCompression(), is(false)); } finally { deleteCore(); } @@ -75,7 +71,7 @@ public void fieldDefinition_halfByteSize_shouldLoadSchemaField() throws Exceptio } @Test - public void fieldDefinition_compressed_shouldLoadSchemaField() throws Exception { + public void fieldDefinition_deprecatedCompress_shouldStillLoadSchemaField() throws Exception { try { initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml"); @@ -87,14 +83,14 @@ public void fieldDefinition_compressed_shouldLoadSchemaField() throws Exception ScalarQuantizedDenseVectorField vectorType = (ScalarQuantizedDenseVectorField) vectorField.getType(); assertThat(vectorType.getBits(), is(4)); - assertThat(vectorType.useCompression(), is(true)); } finally { deleteCore(); } } @Test - public void fieldDefinition_customConfidenceInterval_shouldLoadSchemaField() throws Exception { + public void fieldDefinition_deprecatedConfidenceInterval_shouldStillLoadSchemaField() + throws Exception { try { initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml"); @@ -105,14 +101,16 @@ public void fieldDefinition_customConfidenceInterval_shouldLoadSchemaField() thr ScalarQuantizedDenseVectorField vectorType = (ScalarQuantizedDenseVectorField) vectorField.getType(); - assertThat(vectorType.getConfidenceInterval(), is(0.91F)); + assertThat(vectorType.getDimension(), is(4)); + assertThat(vectorType.getBits(), is(ScalarQuantizedDenseVectorField.DEFAULT_BITS)); } finally { deleteCore(); } } @Test - public void fieldDefinition_dynamicConfidenceInterval_shouldLoadSchemaField() throws Exception { + public void fieldDefinition_deprecatedDynamicConfidenceInterval_shouldStillLoadSchemaField() + throws Exception { try { initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml"); @@ -123,7 +121,8 @@ public void fieldDefinition_dynamicConfidenceInterval_shouldLoadSchemaField() th ScalarQuantizedDenseVectorField vectorType = (ScalarQuantizedDenseVectorField) vectorField.getType(); - assertThat(vectorType.getConfidenceInterval(), is(0f)); + assertThat(vectorType.getDimension(), is(4)); + assertThat(vectorType.getBits(), is(ScalarQuantizedDenseVectorField.DEFAULT_BITS)); } finally { deleteCore(); }