SOLR-18376: remove ScalarQuantizedDenseVectorField compress and confidenceInterval

changelog/unreleased/SOLR-18376-remove-scalarquantized-compress-confidenceinterval.yml

@@ -0,0 +1,8 @@
+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc
+title: Remove the deprecated ScalarQuantizedDenseVectorField compress and confidenceInterval fields and their accessors useCompression and getConfidenceInterval, since Solr 10.1 no longer used by the underlying Lucene codec; the compress, confidenceInterval and dynamicConfidenceInterval schema params are still accepted and ignored with a deprecation warning, so an existing schema that sets them keeps loading. The public four-argument constructor loses its two corresponding parameters and now takes dimension, similarityFunction, vectorEncoding and bits.
+type: removed
+authors:
+  - name: Serhiy Bzhezytskyy
+links:
+  - name: SOLR-18376
+    url: https://issues.apache.org/jira/browse/SOLR-18376

solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java

@@ -18,7 +18,6 @@
 
 import static java.util.Optional.ofNullable;
 
-import com.google.common.annotations.VisibleForTesting;
 import java.util.Map;
 import org.apache.lucene.codecs.KnnVectorsFormat;
 import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;
@@ -36,32 +35,13 @@ public class ScalarQuantizedDenseVectorField extends DenseVectorField {
       "compress"; // can only be enabled when bits = 4 per Lucene codec spec
 
   static final int DEFAULT_BITS = 7; // use signed byte as default when unspecified
-  static final Float DEFAULT_CONFIDENCE_INTERVAL = null; // use dimension scaled confidence interval
 
   /**
    * Number of bits to use for storage Must be 4 (half-byte) or 7 (signed-byte) per Lucene codec
    * spec
    */
   private int bits;
 
-  /**
-   * Confidence interval to use for scalar quantization Default is calculated as
-   * `1-1/(vector_dimensions + 1)`
-   *
-   * @deprecated Since Solr 10.1. No longer used by the underlying Lucene codec.
-   */
-  @Deprecated(since = "10.1")
-  private Float confidenceInterval;
-
-  /**
-   * When enabled, in conjunction with 4 bit size, will pair values into single bytes for 50%
-   * reduction in memory usage (comes at the cost of some decode speed penalty)
-   *
-   * @deprecated Since Solr 10.1. No longer used by the underlying Lucene codec.
-   */
-  @Deprecated(since = "10.1")
-  private boolean compress;
-
   public ScalarQuantizedDenseVectorField() {
     super();
   }
@@ -70,25 +50,22 @@ public ScalarQuantizedDenseVectorField(
       int dimension,
       VectorSimilarityFunction similarityFunction,
       VectorEncoding vectorEncoding,
-      int bits,
-      Float confidenceInterval,
-      boolean compress) {
+      int bits) {
     super(dimension, similarityFunction, vectorEncoding);
     this.bits = bits;
-    this.confidenceInterval = confidenceInterval;
-    this.compress = compress;
   }
 
   @Override
   public void init(IndexSchema schema, Map<String, String> args) {
     this.bits = ofNullable(args.remove(BITS_PARAM)).map(Integer::parseInt).orElse(DEFAULT_BITS);
 
     // These params ("compress", "confidenceInterval", "dynamicConfidenceInterval") are deprecated
-    // since Solr 10.1. Lucene 10.4's scalar-quantized vector format no longer consumes them
-    // directly. They are parsed for backward compatibility but are no-ops going forward.
+    // since Solr 10.1: Lucene's scalar-quantized vector format no longer consumes them, and as of
+    // 11.0 nothing here retains their values either. They are still consumed from the args - and
+    // only warned about - so that an existing schema setting them keeps loading instead of failing
+    // FieldType.setArgs' "invalid arguments" check.
     String compressStr = args.remove(COMPRESS_PARAM);
     if (compressStr != null) {
-      this.compress = Boolean.parseBoolean(compressStr);
       DeprecationLog.log(
           COMPRESS_PARAM,
           "The '"
@@ -99,20 +76,16 @@ public void init(IndexSchema schema, Map<String, String> args) {
 
     String confidenceIntervalStr = args.remove(CONFIDENCE_INTERVAL_PARAM);
     if (confidenceIntervalStr != null) {
-      this.confidenceInterval = Float.parseFloat(confidenceIntervalStr);
       DeprecationLog.log(
           CONFIDENCE_INTERVAL_PARAM,
           "The '"
               + CONFIDENCE_INTERVAL_PARAM
               + "' parameter for ScalarQuantizedDenseVectorField is deprecated since Solr 10.1"
               + " and will be ignored. Please remove it from your schema.");
-    } else {
-      this.confidenceInterval = DEFAULT_CONFIDENCE_INTERVAL;
     }
 
     String dynamicConfidenceIntervalStr = args.remove(DYNAMIC_CONFIDENCE_INTERVAL_PARAM);
     if (Boolean.parseBoolean(dynamicConfidenceIntervalStr)) {
-      this.confidenceInterval = 0f;
       DeprecationLog.log(
           DYNAMIC_CONFIDENCE_INTERVAL_PARAM,
           "The '"
@@ -156,22 +129,4 @@ public void checkSchemaField(final SchemaField field) throws SolrException {
   public int getBits() {
     return bits;
   }
-
-  /**
-   * @deprecated Since Solr 10.1. No longer used by the underlying Lucene codec.
-   */
-  @Deprecated(since = "10.1")
-  @VisibleForTesting
-  boolean useCompression() {
-    return compress;
-  }
-
-  /**
-   * @deprecated Since Solr 10.1. No longer used by the underlying Lucene codec.
-   */
-  @Deprecated(since = "10.1")
-  @VisibleForTesting
-  Float getConfidenceInterval() {
-    return confidenceInterval;
-  }
 }

solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java

@@ -47,10 +47,6 @@ public void fieldDefinition_default_shouldLoadSchemaField() throws Exception {
       assertThat(defaultVectorType.getDimension(), is(4));
       assertThat(defaultVectorType.getKnnAlgorithm(), is("hnsw"));
       assertThat(defaultVectorType.getBits(), is(ScalarQuantizedDenseVectorField.DEFAULT_BITS));
-      assertThat(
-          defaultVectorType.getConfidenceInterval(),
-          is(ScalarQuantizedDenseVectorField.DEFAULT_CONFIDENCE_INTERVAL));
-      assertThat(defaultVectorType.useCompression(), is(false));
     } finally {
       deleteCore();
     }
@@ -75,7 +71,7 @@ public void fieldDefinition_halfByteSize_shouldLoadSchemaField() throws Exceptio
   }
 
   @Test
-  public void fieldDefinition_compressed_shouldLoadSchemaField() throws Exception {
+  public void fieldDefinition_deprecatedCompress_shouldStillLoadSchemaField() throws Exception {
     try {
       initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml");
 
@@ -87,14 +83,14 @@ public void fieldDefinition_compressed_shouldLoadSchemaField() throws Exception
       ScalarQuantizedDenseVectorField vectorType =
           (ScalarQuantizedDenseVectorField) vectorField.getType();
       assertThat(vectorType.getBits(), is(4));
-      assertThat(vectorType.useCompression(), is(true));
     } finally {
       deleteCore();
     }
   }
 
   @Test
-  public void fieldDefinition_customConfidenceInterval_shouldLoadSchemaField() throws Exception {
+  public void fieldDefinition_deprecatedConfidenceInterval_shouldStillLoadSchemaField()
+      throws Exception {
     try {
       initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml");
 
@@ -105,14 +101,16 @@ public void fieldDefinition_customConfidenceInterval_shouldLoadSchemaField() thr
 
       ScalarQuantizedDenseVectorField vectorType =
           (ScalarQuantizedDenseVectorField) vectorField.getType();
-      assertThat(vectorType.getConfidenceInterval(), is(0.91F));
+      assertThat(vectorType.getDimension(), is(4));
+      assertThat(vectorType.getBits(), is(ScalarQuantizedDenseVectorField.DEFAULT_BITS));
     } finally {
       deleteCore();
     }
   }
 
   @Test
-  public void fieldDefinition_dynamicConfidenceInterval_shouldLoadSchemaField() throws Exception {
+  public void fieldDefinition_deprecatedDynamicConfidenceInterval_shouldStillLoadSchemaField()
+      throws Exception {
     try {
       initCore("solrconfig_codec.xml", "schema-densevector-quantized.xml");
 
@@ -123,7 +121,8 @@ public void fieldDefinition_dynamicConfidenceInterval_shouldLoadSchemaField() th
 
       ScalarQuantizedDenseVectorField vectorType =
           (ScalarQuantizedDenseVectorField) vectorField.getType();
-      assertThat(vectorType.getConfidenceInterval(), is(0f));
+      assertThat(vectorType.getDimension(), is(4));
+      assertThat(vectorType.getBits(), is(ScalarQuantizedDenseVectorField.DEFAULT_BITS));
     } finally {
       deleteCore();
     }