From c2fe0adb9f612dc1433ef4eeee3687bbed74e598 Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Thu, 16 Jul 2026 18:02:58 -0400 Subject: [PATCH 1/6] management hook for graph construction params --- .../jvector/graph/GraphIndexBuilder.java | 240 +++++++++++++++++- .../management/GraphIndexBuilderConfig.java | 154 +++++++++++ .../GraphIndexBuilderConfigMBean.java | 73 ++++++ 3 files changed, 464 insertions(+), 3 deletions(-) create mode 100644 jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfig.java create mode 100644 jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfigMBean.java diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/GraphIndexBuilder.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/GraphIndexBuilder.java index 4139a14b6..f6d481072 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/GraphIndexBuilder.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/GraphIndexBuilder.java @@ -25,6 +25,7 @@ import io.github.jbellis.jvector.graph.similarity.BuildScoreProvider; import io.github.jbellis.jvector.graph.similarity.ScoreFunction; import io.github.jbellis.jvector.graph.similarity.SearchScoreProvider; +import io.github.jbellis.jvector.management.GraphIndexBuilderConfig; import io.github.jbellis.jvector.util.*; import io.github.jbellis.jvector.vector.VectorSimilarityFunction; import io.github.jbellis.jvector.vector.types.VectorFloat; @@ -81,6 +82,36 @@ public class GraphIndexBuilder implements Closeable, Accountable { private final Random rng; + /** + * Reads all the vectors from vector values, builds a graph connecting them by their dense + * ordinals, using the given hyperparameter settings, and returns the resulting graph. + * By default, refineFinalGraph = true. + * + * @param vectorValues the vectors whose relations are represented by the graph - must provide a + * different view over those vectors than the one used to add via addGraphNode. + * @param M – the maximum number of connections a node can have + * @param beamWidth the size of the beam search to use when finding nearest neighbors. + * @param neighborOverflow the ratio of extra neighbors to allow temporarily when inserting a + * node. larger values will build more efficiently, but use more memory. + * @param alpha how aggressive pruning diverse neighbors should be. Set alpha > 1.0 to + * allow longer edges. If alpha = 1.0 then the equivalent of the lowest level of + * an HNSW graph will be created, which is usually not what you want. + */ + public GraphIndexBuilder(RandomAccessVectorValues vectorValues, + VectorSimilarityFunction similarityFunction, + int M, + int beamWidth, + float neighborOverflow, + float alpha) + { + this(BuildScoreProvider.randomAccessScoreProvider(vectorValues, similarityFunction), + vectorValues.dimension(), + M, + beamWidth, + neighborOverflow, + alpha); + } + /** * Reads all the vectors from vector values, builds a graph connecting them by their dense * ordinals, using the given hyperparameter settings, and returns the resulting graph. @@ -96,7 +127,10 @@ public class GraphIndexBuilder implements Closeable, Accountable { * allow longer edges. If alpha = 1.0 then the equivalent of the lowest level of * an HNSW graph will be created, which is usually not what you want. * @param addHierarchy whether we want to add an HNSW-style hierarchy on top of the Vamana index. + * @deprecated Use the equivalent constructor without {@code addHierarchy}; that value is now + * controlled via {@link io.github.jbellis.jvector.management.GraphIndexBuilderConfig}. */ + @Deprecated public GraphIndexBuilder(RandomAccessVectorValues vectorValues, VectorSimilarityFunction similarityFunction, int M, @@ -130,7 +164,10 @@ public GraphIndexBuilder(RandomAccessVectorValues vectorValues, * an HNSW graph will be created, which is usually not what you want. * @param addHierarchy whether we want to add an HNSW-style hierarchy on top of the Vamana index. * @param refineFinalGraph whether we do a second pass over each node in the graph to refine its connections + * @deprecated Use the equivalent constructor without {@code addHierarchy} and {@code refineFinalGraph}; + * those values are now controlled via {@link io.github.jbellis.jvector.management.GraphIndexBuilderConfig}. */ + @Deprecated public GraphIndexBuilder(RandomAccessVectorValues vectorValues, VectorSimilarityFunction similarityFunction, int M, @@ -165,7 +202,10 @@ public GraphIndexBuilder(RandomAccessVectorValues vectorValues, * allow longer edges. If alpha = 1.0 then the equivalent of the lowest level of * an HNSW graph will be created, which is usually not what you want. * @param addHierarchy whether we want to add an HNSW-style hierarchy on top of the Vamana index. + * @deprecated Use the equivalent constructor without {@code addHierarchy}; that value is now + * controlled via {@link io.github.jbellis.jvector.management.GraphIndexBuilderConfig}. */ + @Deprecated public GraphIndexBuilder(BuildScoreProvider scoreProvider, int dimension, int M, @@ -177,6 +217,31 @@ public GraphIndexBuilder(BuildScoreProvider scoreProvider, this(scoreProvider, dimension, M, beamWidth, neighborOverflow, alpha, addHierarchy, true, PhysicalCoreExecutor.pool(), ForkJoinPool.commonPool()); } + /** + * Reads all the vectors from vector values, builds a graph connecting them by their dense + * ordinals, using the given hyperparameter settings, and returns the resulting graph. + * Default executor pools are used. + * By default, refineFinalGraph = true. + * + * @param scoreProvider describes how to determine the similarities between vectors + * @param M the maximum number of connections a node can have + * @param beamWidth the size of the beam search to use when finding nearest neighbors. + * @param neighborOverflow the ratio of extra neighbors to allow temporarily when inserting a + * node. larger values will build more efficiently, but use more memory. + * @param alpha how aggressive pruning diverse neighbors should be. Set alpha > 1.0 to + * allow longer edges. If alpha = 1.0 then the equivalent of the lowest level of + * an HNSW graph will be created, which is usually not what you want. + */ + public GraphIndexBuilder(BuildScoreProvider scoreProvider, + int dimension, + int M, + int beamWidth, + float neighborOverflow, + float alpha) + { + this(scoreProvider, dimension, M, beamWidth, neighborOverflow, alpha, PhysicalCoreExecutor.pool(), ForkJoinPool.commonPool()); + } + /** * Reads all the vectors from vector values, builds a graph connecting them by their dense * ordinals, using the given hyperparameter settings, and returns the resulting graph. @@ -192,7 +257,10 @@ public GraphIndexBuilder(BuildScoreProvider scoreProvider, * an HNSW graph will be created, which is usually not what you want. * @param addHierarchy whether we want to add an HNSW-style hierarchy on top of the Vamana index. * @param refineFinalGraph whether we do a second pass over each node in the graph to refine its connections + * @deprecated Use the equivalent constructor without {@code addHierarchy} and {@code refineFinalGraph}; + * those values are now controlled via {@link io.github.jbellis.jvector.management.GraphIndexBuilderConfig}. */ + @Deprecated public GraphIndexBuilder(BuildScoreProvider scoreProvider, int dimension, int M, @@ -222,7 +290,10 @@ public GraphIndexBuilder(BuildScoreProvider scoreProvider, * @param simdExecutor ForkJoinPool instance for SIMD operations, best is to use a pool with the size of * the number of physical cores. * @param parallelExecutor ForkJoinPool instance for parallel stream operations + * @deprecated Use the equivalent constructor without {@code addHierarchy} and {@code refineFinalGraph}; + * those values are now controlled via {@link io.github.jbellis.jvector.management.GraphIndexBuilderConfig}. */ + @Deprecated public GraphIndexBuilder(BuildScoreProvider scoreProvider, int dimension, int M, @@ -237,6 +308,34 @@ public GraphIndexBuilder(BuildScoreProvider scoreProvider, this(scoreProvider, dimension, List.of(M), beamWidth, neighborOverflow, alpha, addHierarchy, refineFinalGraph, simdExecutor, parallelExecutor); } + /** + * Reads all the vectors from vector values, builds a graph connecting them by their dense + * ordinals, using the given hyperparameter settings, and returns the resulting graph. + * + * @param scoreProvider describes how to determine the similarities between vectors + * @param M the maximum number of connections a node can have + * @param beamWidth the size of the beam search to use when finding nearest neighbors. + * @param neighborOverflow the ratio of extra neighbors to allow temporarily when inserting a + * node. larger values will build more efficiently, but use more memory. + * @param alpha how aggressive pruning diverse neighbors should be. Set alpha > 1.0 to + * allow longer edges. If alpha = 1.0 then the equivalent of the lowest level of + * an HNSW graph will be created, which is usually not what you want. + * @param simdExecutor ForkJoinPool instance for SIMD operations, best is to use a pool with the size of + * the number of physical cores. + * @param parallelExecutor ForkJoinPool instance for parallel stream operations + */ + public GraphIndexBuilder(BuildScoreProvider scoreProvider, + int dimension, + int M, + int beamWidth, + float neighborOverflow, + float alpha, + ForkJoinPool simdExecutor, + ForkJoinPool parallelExecutor) + { + this(scoreProvider, dimension, List.of(M), beamWidth, neighborOverflow, alpha, simdExecutor, parallelExecutor); + } + /** * Reads all the vectors from vector values, builds a graph connecting them by their dense * ordinals, using the given hyperparameter settings, and returns the resulting graph. @@ -253,7 +352,10 @@ public GraphIndexBuilder(BuildScoreProvider scoreProvider, * an HNSW graph will be created, which is usually not what you want. * @param addHierarchy whether we want to add an HNSW-style hierarchy on top of the Vamana index. * @param refineFinalGraph whether we do a second pass over each node in the graph to refine its connections + * @deprecated Use the equivalent constructor without {@code addHierarchy} and {@code refineFinalGraph}; + * those values are now controlled via {@link io.github.jbellis.jvector.management.GraphIndexBuilderConfig}. */ + @Deprecated public GraphIndexBuilder(BuildScoreProvider scoreProvider, int dimension, List maxDegrees, @@ -266,6 +368,31 @@ public GraphIndexBuilder(BuildScoreProvider scoreProvider, this(scoreProvider, dimension, maxDegrees, beamWidth, neighborOverflow, alpha, addHierarchy, refineFinalGraph, PhysicalCoreExecutor.pool(), ForkJoinPool.commonPool()); } + /** + * Reads all the vectors from vector values, builds a graph connecting them by their dense + * ordinals, using the given hyperparameter settings, and returns the resulting graph. + * Default executor pools are used. + * + * @param scoreProvider describes how to determine the similarities between vectors + * @param maxDegrees the maximum number of connections a node can have in each layer; if fewer entries + * * are specified than the number of layers, the last entry is used for all remaining layers. + * @param beamWidth the size of the beam search to use when finding nearest neighbors. + * @param neighborOverflow the ratio of extra neighbors to allow temporarily when inserting a + * node. larger values will build more efficiently, but use more memory. + * @param alpha how aggressive pruning diverse neighbors should be. Set alpha > 1.0 to + * allow longer edges. If alpha = 1.0 then the equivalent of the lowest level of + * an HNSW graph will be created, which is usually not what you want. + */ + public GraphIndexBuilder(BuildScoreProvider scoreProvider, + int dimension, + List maxDegrees, + int beamWidth, + float neighborOverflow, + float alpha) + { + this(scoreProvider, dimension, maxDegrees, beamWidth, neighborOverflow, alpha, PhysicalCoreExecutor.pool(), ForkJoinPool.commonPool()); + } + /** * Reads all the vectors from vector values, builds a graph connecting them by their dense * ordinals, using the given hyperparameter settings, and returns the resulting graph. @@ -284,7 +411,10 @@ public GraphIndexBuilder(BuildScoreProvider scoreProvider, * @param simdExecutor ForkJoinPool instance for SIMD operations, best is to use a pool with the size of * the number of physical cores. * @param parallelExecutor ForkJoinPool instance for parallel stream operations + * @deprecated Use the equivalent constructor without {@code addHierarchy} and {@code refineFinalGraph}; + * those values are now controlled via {@link io.github.jbellis.jvector.management.GraphIndexBuilderConfig}. */ + @Deprecated public GraphIndexBuilder(BuildScoreProvider scoreProvider, int dimension, List maxDegrees, @@ -296,6 +426,55 @@ public GraphIndexBuilder(BuildScoreProvider scoreProvider, ForkJoinPool simdExecutor, ForkJoinPool parallelExecutor) { + this(scoreProvider, dimension, maxDegrees, beamWidth, neighborOverflow, alpha, + logCallerAddHierarchy(addHierarchy), + logCallerRefineFinalGraph(refineFinalGraph), + simdExecutor, parallelExecutor, null); + } + + /** + * Reads all the vectors from vector values, builds a graph connecting them by their dense + * ordinals, using the given hyperparameter settings, and returns the resulting graph. + * + * @param scoreProvider describes how to determine the similarities between vectors + * @param maxDegrees the maximum number of connections a node can have in each layer; if fewer entries + * are specified than the number of layers, the last entry is used for all remaining layers. + * @param beamWidth the size of the beam search to use when finding nearest neighbors. + * @param neighborOverflow the ratio of extra neighbors to allow temporarily when inserting a + * node. larger values will build more efficiently, but use more memory. + * @param alpha how aggressive pruning diverse neighbors should be. Set alpha > 1.0 to + * allow longer edges. If alpha = 1.0 then the equivalent of the lowest level of + * an HNSW graph will be created, which is usually not what you want. + * @param simdExecutor ForkJoinPool instance for SIMD operations, best is to use a pool with the size of + * the number of physical cores. + * @param parallelExecutor ForkJoinPool instance for parallel stream operations + */ + public GraphIndexBuilder(BuildScoreProvider scoreProvider, + int dimension, + List maxDegrees, + int beamWidth, + float neighborOverflow, + float alpha, + ForkJoinPool simdExecutor, + ForkJoinPool parallelExecutor) { + this(scoreProvider, dimension, maxDegrees, beamWidth, neighborOverflow, alpha, + resolveJmxAddHierarchy(maxDegrees), + resolveJmxRefineFinalGraph(), + simdExecutor, parallelExecutor, null); + } + + // Private workhorse — all public constructors funnel here. + private GraphIndexBuilder(BuildScoreProvider scoreProvider, + int dimension, + List maxDegrees, + int beamWidth, + float neighborOverflow, + float alpha, + boolean addHierarchy, + boolean refineFinalGraph, + ForkJoinPool simdExecutor, + ForkJoinPool parallelExecutor, + @SuppressWarnings("unused") Void disambiguator) { if (maxDegrees.stream().anyMatch(i -> i <= 0)) { throw new IllegalArgumentException("layer degrees must be positive"); } @@ -312,12 +491,12 @@ public GraphIndexBuilder(BuildScoreProvider scoreProvider, throw new IllegalArgumentException("alpha must be positive"); } + this.addHierarchy = addHierarchy; + this.refineFinalGraph = refineFinalGraph; this.scoreProvider = scoreProvider; this.dimension = dimension; this.neighborOverflow = neighborOverflow; this.alpha = alpha; - this.addHierarchy = addHierarchy; - this.refineFinalGraph = refineFinalGraph; this.beamWidth = beamWidth; this.simdExecutor = simdExecutor; this.parallelExecutor = parallelExecutor; @@ -337,6 +516,33 @@ public GraphIndexBuilder(BuildScoreProvider scoreProvider, this.rng = new Random(0); } + // ── Source-logging helpers ──────────────────────────────────────────────── + // These are evaluated as arguments before this() fires, allowing us to log + // the value source before the constructor body runs. + + private static boolean resolveJmxAddHierarchy(List maxDegrees) { + // if multiple degrees are specified, hierarchy is structurally required + boolean v = maxDegrees.size() > 1 || GraphIndexBuilderConfig.getInstance().isAddHierarchy(); + logger.debug("addHierarchy={} (from GraphIndexBuilderConfig)", v); + return v; + } + + private static boolean resolveJmxRefineFinalGraph() { + boolean v = GraphIndexBuilderConfig.getInstance().isRefineFinalGraph(); + logger.debug("refineFinalGraph={} (from GraphIndexBuilderConfig)", v); + return v; + } + + private static boolean logCallerAddHierarchy(boolean v) { + logger.debug("addHierarchy={} (caller-provided via deprecated constructor)", v); + return v; + } + + private static boolean logCallerRefineFinalGraph(boolean v) { + logger.debug("refineFinalGraph={} (caller-provided via deprecated constructor)", v); + return v; + } + /** * Create this builder from an existing {@link io.github.jbellis.jvector.graph.disk.OnDiskGraphIndex}, this is useful when we just loaded a graph from disk * copy it into {@link OnHeapGraphIndex} and then start mutating it with minimal overhead of recreating the mutable {@link OnHeapGraphIndex} used in the new GraphIndexBuilder object @@ -349,9 +555,37 @@ public GraphIndexBuilder(BuildScoreProvider scoreProvider, * @param refineFinalGraph whether to perform a refinement step on the final graph structure. * @param simdExecutor the ForkJoinPool executor used for SIMD tasks during graph building. * @param parallelExecutor the ForkJoinPool executor used for general parallelization during graph building. + * @deprecated Use the equivalent constructor without {@code refineFinalGraph}; that value is now + * controlled via {@link io.github.jbellis.jvector.management.GraphIndexBuilderConfig}. */ + @Deprecated @Experimental public GraphIndexBuilder(BuildScoreProvider buildScoreProvider, int dimension, MutableGraphIndex mutableGraphIndex, int beamWidth, float neighborOverflow, float alpha, boolean refineFinalGraph, ForkJoinPool simdExecutor, ForkJoinPool parallelExecutor) { + this(buildScoreProvider, dimension, mutableGraphIndex, beamWidth, neighborOverflow, alpha, + logCallerRefineFinalGraph(refineFinalGraph), simdExecutor, parallelExecutor, + null); + } + + /** + * Create this builder from an existing {@link io.github.jbellis.jvector.graph.disk.OnDiskGraphIndex}, this is useful when we just loaded a graph from disk + * copy it into {@link OnHeapGraphIndex} and then start mutating it with minimal overhead of recreating the mutable {@link OnHeapGraphIndex} used in the new GraphIndexBuilder object + * + * @param buildScoreProvider the provider responsible for calculating build scores. + * @param mutableGraphIndex a mutable graph index. + * @param beamWidth the width of the beam used during the graph building process. + * @param neighborOverflow the factor determining how many additional neighbors are allowed beyond the configured limit. + * @param alpha the weight factor for balancing score computations. + * @param simdExecutor the ForkJoinPool executor used for SIMD tasks during graph building. + * @param parallelExecutor the ForkJoinPool executor used for general parallelization during graph building. + */ + public GraphIndexBuilder(BuildScoreProvider buildScoreProvider, int dimension, MutableGraphIndex mutableGraphIndex, int beamWidth, float neighborOverflow, float alpha, ForkJoinPool simdExecutor, ForkJoinPool parallelExecutor) { + this(buildScoreProvider, dimension, mutableGraphIndex, beamWidth, neighborOverflow, alpha, + resolveJmxRefineFinalGraph(), simdExecutor, parallelExecutor, + null); + } + + // Private mutableGraphIndex workhorse — addHierarchy is always derived from the existing graph. + private GraphIndexBuilder(BuildScoreProvider buildScoreProvider, int dimension, MutableGraphIndex mutableGraphIndex, int beamWidth, float neighborOverflow, float alpha, boolean refineFinalGraph, ForkJoinPool simdExecutor, ForkJoinPool parallelExecutor, @SuppressWarnings("unused") Void disambiguator) { if (beamWidth <= 0) { throw new IllegalArgumentException("beamWidth must be positive"); } @@ -366,6 +600,7 @@ public GraphIndexBuilder(BuildScoreProvider buildScoreProvider, int dimension, M this.neighborOverflow = neighborOverflow; this.dimension = dimension; this.alpha = alpha; + // addHierarchy is structural — it must match the existing graph's topology this.addHierarchy = mutableGraphIndex.isHierarchical(); this.refineFinalGraph = refineFinalGraph; this.beamWidth = beamWidth; @@ -1063,7 +1298,6 @@ public static ImmutableGraphIndex buildAndMergeNewNodes(RandomAccessReader in, beamWidth, overflowRatio, alpha, - true, simdExecutor, parallelExecutor ); diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfig.java b/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfig.java new file mode 100644 index 000000000..f65bd38ba --- /dev/null +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfig.java @@ -0,0 +1,154 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.github.jbellis.jvector.management; + +import io.github.jbellis.jvector.annotations.Experimental; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.management.MBeanServer; +import javax.management.ObjectName; +import java.lang.management.ManagementFactory; + +/** + * Singleton that holds JMX-managed default values for + * {@link io.github.jbellis.jvector.graph.GraphIndexBuilder} construction parameters. + * + *

JMX Pattern — Standard MBean

+ * + *

This class uses Java's Standard MBean pattern, the simplest form of JMX + * management. The rules are: + *

    + *
  1. Define an interface whose name ends in {@code MBean} + * ({@link GraphIndexBuilderConfigMBean}).
    2. + *
  2. Implement that interface in a class with the same name minus the {@code MBean} + * suffix (this class).
    3. + *
  3. Register an instance with the platform {@link MBeanServer} under a unique + * {@link ObjectName}.
    4. + *
+ * + *

Once registered, any JMX client can inspect and modify the exposed attributes. + * For example, using JConsole: + * 

+ *   MBeans → io.github.jbellis.jvector → GraphIndexBuilderConfig → Attributes
+ *       AddHierarchy : true   ← current value
+ *                    [edit to false and press Enter to apply]
+ *
+ * + * Or programmatically via {@code jmxterm}: + * 
+ *   open <pid>
+ *   bean io.github.jbellis.jvector:type=GraphIndexBuilderConfig
+ *   get AddHierarchy
+ *   set AddHierarchy false
+ *
+ * + *

Usage

+ * + *

Code that creates a {@code GraphIndexBuilder} and wants to respect the JMX-managed + * value reads from the singleton before construction: + * 

{@code
+ * boolean addHierarchy = GraphIndexBuilderConfig.getInstance().isAddHierarchy();
+ * var builder = new GraphIndexBuilder(scoreProvider, dimension, M, beamWidth,
+ *                                     neighborOverflow, alpha, addHierarchy);
+ * }
+ * + *

Thread Safety

+ * + *

All managed attributes are stored as {@code volatile} fields so that writes from a + * JMX thread are immediately visible to application threads without additional + * synchronization. + * + *

Failure Policy

+ * + *

MBean registration is performed in the constructor and wrapped in a try/catch. + * Registration failure (e.g., because the JVM has no platform MBeanServer or the name + * is already taken) logs a warning and is otherwise silently ignored — the singleton is + * still usable with its default values, so JMX availability is never on the critical + * path. + */ +@Experimental +public class GraphIndexBuilderConfig implements GraphIndexBuilderConfigMBean { + + private static final Logger logger = LoggerFactory.getLogger(GraphIndexBuilderConfig.class); + + /** + * JMX ObjectName under which this MBean is registered. + * Domain: project base package. Type: simple class name. + */ + public static final String OBJECT_NAME = "io.github.jbellis.jvector:type=GraphIndexBuilderConfig"; + + // ── Singleton ──────────────────────────────────────────────────────────── + // Initialized at class-load time; the JVM guarantees exactly-once, thread-safe + // initialization of static fields. + private static final GraphIndexBuilderConfig INSTANCE = new GraphIndexBuilderConfig(); + + public static GraphIndexBuilderConfig getInstance() { + return INSTANCE; + } + + // ── Managed attributes ─────────────────────────────────────────────────── + // volatile ensures writes by a JMX client thread are immediately visible + // to any thread that subsequently reads the field. + + private volatile boolean addHierarchy = true; + private volatile boolean refineFinalGraph = true; + + // ── Constructor ────────────────────────────────────────────────────────── + + private GraphIndexBuilderConfig() { + try { + MBeanServer server = ManagementFactory.getPlatformMBeanServer(); + ObjectName name = new ObjectName(OBJECT_NAME); + server.registerMBean(this, name); + logger.info("Registered JMX MBean: {}", OBJECT_NAME); + } catch (Exception e) { + // JMX registration is best-effort; do not disrupt normal operation. + logger.warn("Failed to register JMX MBean '{}': {}", OBJECT_NAME, e.getMessage()); + } + } + + // ── GraphIndexBuilderConfigMBean ───────────────────────────────────────── + + @Override + public boolean isAddHierarchy() { + return addHierarchy; + } + + @Override + public void setAddHierarchy(boolean addHierarchy) { + boolean previous = this.addHierarchy; + this.addHierarchy = addHierarchy; + if (previous != addHierarchy) { + logger.info("JMX: addHierarchy changed {} → {}", previous, addHierarchy); + } + } + + @Override + public boolean isRefineFinalGraph() { + return refineFinalGraph; + } + + @Override + public void setRefineFinalGraph(boolean refineFinalGraph) { + boolean previous = this.refineFinalGraph; + this.refineFinalGraph = refineFinalGraph; + if (previous != refineFinalGraph) { + logger.info("JMX: refineFinalGraph changed {} → {}", previous, refineFinalGraph); + } + } +} diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfigMBean.java b/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfigMBean.java new file mode 100644 index 000000000..acfbf6556 --- /dev/null +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfigMBean.java @@ -0,0 +1,73 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.github.jbellis.jvector.management; + +/** + * JMX Standard MBean interface for {@link GraphIndexBuilderConfig}. + * + *

Exposes {@link io.github.jbellis.jvector.graph.GraphIndexBuilder} construction + * parameters as JMX-managed attributes so they can be inspected and updated at runtime + * via any JMX client (JConsole, jvisualvm, jmxterm, etc.) without restarting the + * application. + * + *

Changes to these attributes take effect the next time a + * {@link io.github.jbellis.jvector.graph.GraphIndexBuilder} reads the value from + * {@link GraphIndexBuilderConfig#getInstance()}. They do not affect indexes that are + * already being built or have already been built. + * + *

The interface follows the Standard MBean naming convention: the implementation + * class ({@link GraphIndexBuilderConfig}) has the same simple name as this interface + * without the {@code MBean} suffix. + */ +public interface GraphIndexBuilderConfigMBean { + + // ── Graph topology ──────────────────────────────────────────────────────── + + /** + * Returns whether HNSW-style hierarchy layers are added on top of the base Vamana + * graph during index construction. + * + *

When {@code true}, the graph has multiple levels (like HNSW), which improves + * search speed on large datasets by reducing the number of distance computations + * needed to reach the entry point region. When {@code false}, only the flat + * level-0 graph is built (equivalent to a plain Vamana index), which uses less + * memory and may build faster on small datasets. + */ + boolean isAddHierarchy(); + + /** + * Enables or disables HNSW-style hierarchy layers for subsequent index builds. + * + * @param addHierarchy {@code true} to enable hierarchy (default), {@code false} to disable + */ + void setAddHierarchy(boolean addHierarchy); + + /** + * Returns whether a second refinement pass is run over each node's edges after + * the initial graph build completes. + * + *

Refinement improves recall at the cost of additional build time. + */ + boolean isRefineFinalGraph(); + + /** + * Enables or disables the final graph refinement pass. + * + * @param refineFinalGraph {@code true} to enable refinement (default), {@code false} to skip + */ + void setRefineFinalGraph(boolean refineFinalGraph); +} From b6edb80489f1647af045ea47fa3bdd5b19ca98a9 Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Fri, 17 Jul 2026 11:08:20 -0400 Subject: [PATCH 2/6] adding boolean switch for parallel writes --- .../jvector/graph/GraphIndexBuilder.java | 5 + .../RandomAccessOnDiskGraphIndexWriter.java | 74 ++++++ .../management/GraphIndexBuilderConfig.java | 15 ++ .../GraphIndexBuilderConfigMBean.java | 21 ++ .../github/jbellis/jvector/example/Grid.java | 10 +- .../graph/TestGraphIndexBuilderConfig.java | 240 ++++++++++++++++++ 6 files changed, 360 insertions(+), 5 deletions(-) create mode 100644 jvector-tests/src/test/java/io/github/jbellis/jvector/graph/TestGraphIndexBuilderConfig.java diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/GraphIndexBuilder.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/GraphIndexBuilder.java index f6d481072..d4df1a9bf 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/GraphIndexBuilder.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/GraphIndexBuilder.java @@ -71,6 +71,11 @@ public class GraphIndexBuilder implements Closeable, Accountable { @VisibleForTesting final MutableGraphIndex graph; + @VisibleForTesting + boolean isRefineFinalGraph() { + return refineFinalGraph; + } + private final ConcurrentSkipListSet insertionsInProgress = new ConcurrentSkipListSet<>(); private final BuildScoreProvider scoreProvider; diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/RandomAccessOnDiskGraphIndexWriter.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/RandomAccessOnDiskGraphIndexWriter.java index 6cd8d0010..221a2b620 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/RandomAccessOnDiskGraphIndexWriter.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/disk/RandomAccessOnDiskGraphIndexWriter.java @@ -16,15 +16,22 @@ package io.github.jbellis.jvector.graph.disk; +import io.github.jbellis.jvector.disk.BufferedRandomAccessWriter; import io.github.jbellis.jvector.disk.RandomAccessWriter; import io.github.jbellis.jvector.graph.ImmutableGraphIndex; import io.github.jbellis.jvector.graph.OnHeapGraphIndex; import io.github.jbellis.jvector.graph.disk.feature.Feature; import io.github.jbellis.jvector.graph.disk.feature.FeatureId; +import io.github.jbellis.jvector.management.GraphIndexBuilderConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.file.Path; import java.util.EnumMap; import java.util.Map; +import java.util.concurrent.ExecutorService; import java.util.function.IntFunction; /** @@ -40,6 +47,8 @@ * */ public abstract class RandomAccessOnDiskGraphIndexWriter extends AbstractGraphIndexWriter { + private static final Logger logger = LoggerFactory.getLogger(RandomAccessOnDiskGraphIndexWriter.class); + protected final long startOffset; /** @@ -178,6 +187,71 @@ public synchronized void write(Map> featur protected abstract void writeL0Records(ImmutableGraphIndex.View view, Map> featureStateSuppliers) throws IOException; + /** + * Unified builder for {@link RandomAccessOnDiskGraphIndexWriter}. + * + *

Reads {@link GraphIndexBuilderConfig#isParallelBuild()} at {@link #build()} time to decide + * whether to instantiate an {@link OnDiskParallelGraphIndexWriter} (parallel L0 serialisation + * via {@code AsynchronousFileChannel}) or an {@link OnDiskGraphIndexWriter} (sequential). + * Both produce an identical on-disk format. + * + *

Parallel-specific options ({@link #withParallelWorkerThreads}, {@link #withParallelDirectBuffers}, + * {@link #withExecutor}) are accepted unconditionally but silently ignored when the sequential + * writer is selected. + */ + public static class Builder extends AbstractGraphIndexWriter.Builder { + private long startOffset = 0L; + private final Path filePath; + private int parallelWorkerThreads = 0; + private boolean parallelUseDirectBuffers = false; + private ExecutorService parallelExecutor = null; + + public Builder(ImmutableGraphIndex graphIndex, Path outPath) throws FileNotFoundException { + super(graphIndex, new BufferedRandomAccessWriter(outPath)); + this.filePath = outPath; + } + + public Builder withStartOffset(long startOffset) { + this.startOffset = startOffset; + return this; + } + + /** Number of worker threads for parallel L0 writes (0 = available processors). Ignored in sequential mode. */ + public Builder withParallelWorkerThreads(int workerThreads) { + this.parallelWorkerThreads = workerThreads; + return this; + } + + /** Whether to use direct {@code ByteBuffer}s for parallel L0 writes. Ignored in sequential mode. */ + public Builder withParallelDirectBuffers(boolean useDirectBuffers) { + this.parallelUseDirectBuffers = useDirectBuffers; + return this; + } + + /** + * Caller-supplied executor for parallel L0 writes; must outlive the writer and is the + * caller's to shut down. Ignored in sequential mode. + */ + public Builder withExecutor(ExecutorService executor) { + this.parallelExecutor = executor; + return this; + } + + @Override + protected RandomAccessOnDiskGraphIndexWriter reallyBuild(int dimension) { + if (GraphIndexBuilderConfig.getInstance().isParallelBuild()) { + logger.debug("graph index write path: parallel (OnDiskParallelGraphIndexWriter)"); + return new OnDiskParallelGraphIndexWriter(out, version, startOffset, graphIndex, + ordinalMapper, dimension, features, filePath, + parallelWorkerThreads, parallelUseDirectBuffers, parallelExecutor); + } else { + logger.debug("graph index write path: sequential (OnDiskGraphIndexWriter)"); + return new OnDiskGraphIndexWriter(out, version, startOffset, graphIndex, + ordinalMapper, dimension, features); + } + } + } + /** * Computes the file offset for the inline features of a given ordinal. * diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfig.java b/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfig.java index f65bd38ba..832e690ae 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfig.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfig.java @@ -107,6 +107,7 @@ public static GraphIndexBuilderConfig getInstance() { private volatile boolean addHierarchy = true; private volatile boolean refineFinalGraph = true; + private volatile boolean parallelBuild = false; // ── Constructor ────────────────────────────────────────────────────────── @@ -151,4 +152,18 @@ public void setRefineFinalGraph(boolean refineFinalGraph) { logger.info("JMX: refineFinalGraph changed {} → {}", previous, refineFinalGraph); } } + + @Override + public boolean isParallelBuild() { + return parallelBuild; + } + + @Override + public void setParallelBuild(boolean parallelBuild) { + boolean previous = this.parallelBuild; + this.parallelBuild = parallelBuild; + if (previous != parallelBuild) { + logger.info("JMX: parallelBuild changed {} → {}", previous, parallelBuild); + } + } } diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfigMBean.java b/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfigMBean.java index acfbf6556..917cd2fee 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfigMBean.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfigMBean.java @@ -70,4 +70,25 @@ public interface GraphIndexBuilderConfigMBean { * @param refineFinalGraph {@code true} to enable refinement (default), {@code false} to skip */ void setRefineFinalGraph(boolean refineFinalGraph); + + // ── Write path ──────────────────────────────────────────────────────────── + + /** + * Returns whether graph index writes use the parallel writer + * ({@link io.github.jbellis.jvector.graph.disk.OnDiskParallelGraphIndexWriter}) or the + * sequential writer ({@link io.github.jbellis.jvector.graph.disk.OnDiskGraphIndexWriter}). + * + *

The parallel writer serialises level-0 records concurrently via an + * {@code AsynchronousFileChannel}, which substantially reduces wall-clock write time for + * large indexes. Both writers produce an identical on-disk format; switching this flag + * does not require re-reading or re-indexing existing data. + */ + boolean isParallelBuild(); + + /** + * Enables or disables the parallel graph index writer for subsequent builds. + * + * @param parallelBuild {@code true} to use the parallel writer, {@code false} for sequential (default) + */ + void setParallelBuild(boolean parallelBuild); } diff --git a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java index 8f45df2a0..04b82dcfe 100644 --- a/jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java +++ b/jvector-examples/src/main/java/io/github/jbellis/jvector/example/Grid.java @@ -388,9 +388,9 @@ private static Map, ImmutableGraphIndex> buildOnDisk(List, OnDiskGraphIndexWriter> writers = new HashMap<>(); + Map, RandomAccessOnDiskGraphIndexWriter> writers = new HashMap<>(); Map, Map>> suppliers = new HashMap<>(); - OnDiskGraphIndexWriter scoringWriter = null; + RandomAccessOnDiskGraphIndexWriter scoringWriter = null; int n = 0; for (var features : featureSets) { // if we are using index caching, use cache names instead of tmp names for index files.... @@ -487,7 +487,7 @@ private static BuilderWithSuppliers builderWithSuppliers(Set features throws FileNotFoundException { var identityMapper = new OrdinalMapper.IdentityMapper(floatVectors.size() - 1); - var builder = new OnDiskGraphIndexWriter.Builder(onHeapGraph, outPath); + var builder = new RandomAccessOnDiskGraphIndexWriter.Builder(onHeapGraph, outPath); builder.withMapper(identityMapper); Map> suppliers = new EnumMap<>(FeatureId.class); @@ -539,10 +539,10 @@ private static DiagnosticLevel getDiagnosticLevel() { } private static class BuilderWithSuppliers { - public final OnDiskGraphIndexWriter.Builder builder; + public final RandomAccessOnDiskGraphIndexWriter.Builder builder; public final Map> suppliers; - public BuilderWithSuppliers(OnDiskGraphIndexWriter.Builder builder, Map> suppliers) { + public BuilderWithSuppliers(RandomAccessOnDiskGraphIndexWriter.Builder builder, Map> suppliers) { this.builder = builder; this.suppliers = suppliers; } diff --git a/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/TestGraphIndexBuilderConfig.java b/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/TestGraphIndexBuilderConfig.java new file mode 100644 index 000000000..2835df5fb --- /dev/null +++ b/jvector-tests/src/test/java/io/github/jbellis/jvector/graph/TestGraphIndexBuilderConfig.java @@ -0,0 +1,240 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.github.jbellis.jvector.graph; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import io.github.jbellis.jvector.LuceneTestCase; +import io.github.jbellis.jvector.TestUtil; +import io.github.jbellis.jvector.disk.SimpleMappedReader; +import io.github.jbellis.jvector.graph.disk.OnDiskGraphIndex; +import io.github.jbellis.jvector.graph.disk.RandomAccessOnDiskGraphIndexWriter; +import io.github.jbellis.jvector.graph.disk.feature.Feature; +import io.github.jbellis.jvector.graph.disk.feature.FeatureId; +import io.github.jbellis.jvector.graph.disk.feature.InlineVectors; +import io.github.jbellis.jvector.management.GraphIndexBuilderConfig; +import io.github.jbellis.jvector.vector.VectorSimilarityFunction; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; + +/** + * Verifies that GraphIndexBuilderConfig (JMX) values are correctly routed through + * the non-deprecated constructor path, and that deprecated constructors continue to + * honour their caller-supplied values without reading JMX. + * + * Covers: + * - addHierarchy: deprecated (old path) and JMX (new path), both true and false + * - refineFinalGraph: deprecated (old path) and JMX (new path), both true and false + * - parallelBuild: unified RandomAccessOnDiskGraphIndexWriter.Builder, both serial and parallel + */ +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class TestGraphIndexBuilderConfig extends LuceneTestCase { + + private static final int DIMENSION = 16; + private static final int SIZE = 200; + private static final int M = 16; + private static final int BEAM_WIDTH = 100; + private static final float NEIGHBOR_OVERFLOW = 1.2f; + private static final float ALPHA = 1.2f; + + private Path testDirectory; + private boolean savedAddHierarchy; + private boolean savedRefineFinalGraph; + private boolean savedParallelBuild; + + @Before + public void setup() throws IOException { + testDirectory = Files.createTempDirectory(getClass().getSimpleName()); + var config = GraphIndexBuilderConfig.getInstance(); + savedAddHierarchy = config.isAddHierarchy(); + savedRefineFinalGraph = config.isRefineFinalGraph(); + savedParallelBuild = config.isParallelBuild(); + } + + @After + public void tearDown() throws Exception { + TestUtil.deleteQuietly(testDirectory); + var config = GraphIndexBuilderConfig.getInstance(); + config.setAddHierarchy(savedAddHierarchy); + config.setRefineFinalGraph(savedRefineFinalGraph); + config.setParallelBuild(savedParallelBuild); + } + + // ── addHierarchy ────────────────────────────────────────────────────────── + + @Test + @SuppressWarnings("deprecation") + public void testAddHierarchy_deprecated_false() { + var ravv = buildVectors(); + var builder = new GraphIndexBuilder(ravv, VectorSimilarityFunction.COSINE, + M, BEAM_WIDTH, NEIGHBOR_OVERFLOW, ALPHA, false); + TestUtil.buildSequentially(builder, ravv); + assertEquals(0, ((OnHeapGraphIndex) builder.graph).getMaxLevel()); + } + + @Test + @SuppressWarnings("deprecation") + public void testAddHierarchy_deprecated_true() { + var ravv = buildVectors(); + var builder = new GraphIndexBuilder(ravv, VectorSimilarityFunction.COSINE, + M, BEAM_WIDTH, NEIGHBOR_OVERFLOW, ALPHA, true); + TestUtil.buildSequentially(builder, ravv); + assertTrue(((OnHeapGraphIndex) builder.graph).getMaxLevel() > 0); + } + + @Test + public void testAddHierarchy_jmx_false() { + GraphIndexBuilderConfig.getInstance().setAddHierarchy(false); + var ravv = buildVectors(); + var builder = new GraphIndexBuilder(ravv, VectorSimilarityFunction.COSINE, + M, BEAM_WIDTH, NEIGHBOR_OVERFLOW, ALPHA); + TestUtil.buildSequentially(builder, ravv); + assertEquals(0, ((OnHeapGraphIndex) builder.graph).getMaxLevel()); + } + + @Test + public void testAddHierarchy_jmx_true() { + GraphIndexBuilderConfig.getInstance().setAddHierarchy(true); + var ravv = buildVectors(); + var builder = new GraphIndexBuilder(ravv, VectorSimilarityFunction.COSINE, + M, BEAM_WIDTH, NEIGHBOR_OVERFLOW, ALPHA); + TestUtil.buildSequentially(builder, ravv); + assertTrue(((OnHeapGraphIndex) builder.graph).getMaxLevel() > 0); + } + + // ── refineFinalGraph ────────────────────────────────────────────────────── + + @Test + @SuppressWarnings("deprecation") + public void testRefineFinalGraph_deprecated_false() { + var ravv = buildVectors(); + var builder = new GraphIndexBuilder(ravv, VectorSimilarityFunction.COSINE, + M, BEAM_WIDTH, NEIGHBOR_OVERFLOW, ALPHA, true, false); + assertFalse(builder.isRefineFinalGraph()); + } + + @Test + @SuppressWarnings("deprecation") + public void testRefineFinalGraph_deprecated_true() { + var ravv = buildVectors(); + var builder = new GraphIndexBuilder(ravv, VectorSimilarityFunction.COSINE, + M, BEAM_WIDTH, NEIGHBOR_OVERFLOW, ALPHA, true, true); + assertTrue(builder.isRefineFinalGraph()); + } + + @Test + public void testRefineFinalGraph_jmx_false() { + GraphIndexBuilderConfig.getInstance().setRefineFinalGraph(false); + var ravv = buildVectors(); + var builder = new GraphIndexBuilder(ravv, VectorSimilarityFunction.COSINE, + M, BEAM_WIDTH, NEIGHBOR_OVERFLOW, ALPHA); + assertFalse(builder.isRefineFinalGraph()); + } + + @Test + public void testRefineFinalGraph_jmx_true() { + GraphIndexBuilderConfig.getInstance().setRefineFinalGraph(true); + var ravv = buildVectors(); + var builder = new GraphIndexBuilder(ravv, VectorSimilarityFunction.COSINE, + M, BEAM_WIDTH, NEIGHBOR_OVERFLOW, ALPHA); + assertTrue(builder.isRefineFinalGraph()); + } + + // ── parallel vs. sequential build ───────────────────────────────────────── + + @Test + public void testUnifiedBuilder_sequential() throws IOException { + GraphIndexBuilderConfig.getInstance().setParallelBuild(false); + var ravv = buildVectors(); + var graph = TestUtil.buildSequentially( + new GraphIndexBuilder(ravv, VectorSimilarityFunction.COSINE, M, BEAM_WIDTH, NEIGHBOR_OVERFLOW, ALPHA), + ravv); + writeAndVerify(graph, ravv, testDirectory.resolve("sequential.index")); + } + + @Test + public void testUnifiedBuilder_parallel() throws IOException { + GraphIndexBuilderConfig.getInstance().setParallelBuild(true); + var ravv = buildVectors(); + var graph = TestUtil.buildSequentially( + new GraphIndexBuilder(ravv, VectorSimilarityFunction.COSINE, M, BEAM_WIDTH, NEIGHBOR_OVERFLOW, ALPHA), + ravv); + writeAndVerify(graph, ravv, testDirectory.resolve("parallel.index")); + } + + @Test + public void testUnifiedBuilder_parallelAndSequentialProduceIdenticalGraph() throws IOException { + var ravv = buildVectors(); + var graph = TestUtil.buildSequentially( + new GraphIndexBuilder(ravv, VectorSimilarityFunction.COSINE, M, BEAM_WIDTH, NEIGHBOR_OVERFLOW, ALPHA), + ravv); + + var seqPath = testDirectory.resolve("seq.index"); + var parPath = testDirectory.resolve("par.index"); + + GraphIndexBuilderConfig.getInstance().setParallelBuild(false); + writeGraph(graph, ravv, seqPath); + + GraphIndexBuilderConfig.getInstance().setParallelBuild(true); + writeGraph(graph, ravv, parPath); + + try (var seqSupplier = new SimpleMappedReader.Supplier(seqPath); + var parSupplier = new SimpleMappedReader.Supplier(parPath)) { + var seqLoaded = OnDiskGraphIndex.load(seqSupplier); + var parLoaded = OnDiskGraphIndex.load(parSupplier); + TestUtil.assertGraphEquals(seqLoaded, parLoaded); + } + } + + // ── helpers ─────────────────────────────────────────────────────────────── + + private ListRandomAccessVectorValues buildVectors() { + return new ListRandomAccessVectorValues( + new ArrayList<>(TestUtil.createRandomVectors(SIZE, DIMENSION)), + DIMENSION + ); + } + + private void writeGraph(ImmutableGraphIndex graph, ListRandomAccessVectorValues ravv, Path path) throws IOException { + var suppliers = Feature.singleStateFactory( + FeatureId.INLINE_VECTORS, + nodeId -> new InlineVectors.State(ravv.getVector(nodeId)) + ); + try (var writer = new RandomAccessOnDiskGraphIndexWriter.Builder(graph, path) + .with(new InlineVectors(ravv.dimension())) + .build()) { + writer.write(suppliers); + } + } + + private void writeAndVerify(ImmutableGraphIndex graph, ListRandomAccessVectorValues ravv, Path path) throws IOException { + writeGraph(graph, ravv, path); + try (var readerSupplier = new SimpleMappedReader.Supplier(path)) { + var onDiskGraph = OnDiskGraphIndex.load(readerSupplier); + TestUtil.assertGraphEquals(graph, onDiskGraph); + } + } +} From af0f99fd7e3d8bb9b3892bb2ed6f5b6cbb550983 Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Tue, 21 Jul 2026 10:50:28 -0400 Subject: [PATCH 3/6] adding release notes --- .../4.1.0/replace_with_pr.feature.md | 151 ++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 docs/release notes/4.1.0/replace_with_pr.feature.md diff --git a/docs/release notes/4.1.0/replace_with_pr.feature.md b/docs/release notes/4.1.0/replace_with_pr.feature.md new file mode 100644 index 000000000..b924ddc81 --- /dev/null +++ b/docs/release notes/4.1.0/replace_with_pr.feature.md @@ -0,0 +1,151 @@ +### JMX Runtime Configuration for Graph Index Builder + +**Description** +Introduces `GraphIndexBuilderConfig`, a JMX-managed singleton that exposes `GraphIndexBuilder` +construction parameters as runtime-tunable attributes. Before this change, options such as +`addHierarchy`, `refineFinalGraph`, and `parallelBuild` could only be set at the construction +call site and required a code change or application restart to modify. With this change, all +three parameters can be inspected and updated live via any standard JMX client — JConsole, +jvisualvm, jmxterm, or a monitoring agent — without restarting the JVM. + +Changes take effect the next time a `GraphIndexBuilder` is constructed; they do not affect +indexes that are already being built or have already been built. + +The implementation follows the Standard MBean pattern: `GraphIndexBuilderConfigMBean` declares +the managed attributes and `GraphIndexBuilderConfig` is the singleton implementation registered +under the object name `io.github.jbellis.jvector:type=GraphIndexBuilderConfig`. All attributes +are stored as `volatile` fields so writes from a JMX management thread are immediately visible +to application threads without additional synchronization. MBean registration is best-effort: +a registration failure (for example, in a restricted JVM environment) logs a warning but does +not disrupt normal operation — the singleton continues to supply its default values. + +**Managed Attributes** + +| Attribute | Type | Default | Description | +|---|---|---|---| +| `AddHierarchy` | `boolean` | `true` | When `true`, builds HNSW-style hierarchy layers on top of the base Vamana graph. `false` produces a flat level-0 (plain Vamana) index, which uses less memory and may build faster on small datasets. | +| `RefineFinalGraph` | `boolean` | `true` | When `true`, runs a second diversity-refinement pass over each node's edges after the initial build. Improves recall at the cost of additional build time. | +| `ParallelBuild` | `boolean` | `false` | When `true`, serializes level-0 node records concurrently via `OnDiskParallelGraphIndexWriter`. Both writers produce an identical on-disk format; switching this flag does not require re-indexing existing data. | + +**How to Enable** + +`GraphIndexBuilderConfig` is initialized automatically on first access and registers its MBean +with the platform MBeanServer. No application code changes are required to activate JMX +management — connecting a JMX client to a running JVector process is sufficient. + +*Programmatic access* — read or set values directly from application code: + +```java +import io.github.jbellis.jvector.management.GraphIndexBuilderConfig; + +GraphIndexBuilderConfig config = GraphIndexBuilderConfig.getInstance(); + +// Read current values +boolean addHierarchy = config.isAddHierarchy(); +boolean refineFinal = config.isRefineFinalGraph(); +boolean parallelBuild = config.isParallelBuild(); + +// Update at runtime (affects all subsequent GraphIndexBuilder constructions) +config.setAddHierarchy(false); +config.setParallelBuild(true); +``` + +*Non-deprecated constructor* — `GraphIndexBuilder` constructors that do not accept explicit +boolean flags read from `GraphIndexBuilderConfig` at construction time: + +```java +// Reads addHierarchy and refineFinalGraph from JMX config +var builder = new GraphIndexBuilder(scoreProvider, dimension, M, beamWidth, + neighborOverflow, alpha); +``` + +Constructors that accept explicit flags continue to honor the caller-supplied values and do +not consult the singleton, enabling call-site overrides when needed. + +**Using JConsole** + +JConsole is the standard JMX browser included with every JDK installation. + +1. **Launch JConsole** + + ``` + jconsole + ``` + + In the connection dialog, select the target JVM process by name or PID and click + **Connect**. If connecting to a remote process, use + `:` after enabling remote JMX on the target JVM: + + ``` + -Dcom.sun.management.jmxremote + -Dcom.sun.management.jmxremote.port=9999 + -Dcom.sun.management.jmxremote.authenticate=false + -Dcom.sun.management.jmxremote.ssl=false + ``` + +2. **Navigate to the MBean** + + Select the **MBeans** tab. In the left-hand tree expand: + + ``` + io.github.jbellis.jvector + └── GraphIndexBuilderConfig + └── Attributes + ``` + +3. **Read an attribute** + + Click on **Attributes**. The right-hand panel lists all three attributes with their + current values: + + ``` + AddHierarchy true + RefineFinalGraph true + ParallelBuild false + ``` + +4. **Set an attribute** + + Double-click the value cell next to the attribute you want to change, type the new + value (`true` or `false`), and press **Enter**. The change takes effect immediately; + the next `GraphIndexBuilder` constructed in that JVM will use the new value. + + Attribute changes are also logged at `INFO` level by JVector: + + ``` + INFO GraphIndexBuilderConfig - JMX: addHierarchy changed true → false + ``` + +**Using jmxterm (command-line alternative)** + +```bash +# Connect to the target JVM by PID +java -jar jmxterm.jar +open + +# Navigate to the MBean +bean io.github.jbellis.jvector:type=GraphIndexBuilderConfig + +# Read all attributes +info -b + +# Read a specific attribute +get AddHierarchy + +# Set an attribute +set AddHierarchy false +set ParallelBuild true +``` + +**Notes** + +- The `GraphIndexBuilderConfig` class and its MBean interface are annotated `@Experimental`. + The attribute set and object name may change in a future release. +- JMX attribute changes are global — they apply to all `GraphIndexBuilder` instances created + after the change in the same JVM. Per-graph overrides are not supported in this release; + callers that need per-graph control should pass values explicitly to the appropriate + constructor. +- The `parallelBuild` attribute requires the `OnDiskParallelGraphIndexWriter` to be on the + classpath (it is part of `jvector-base`). When `parallelBuild` is `true`, the unified + `RandomAccessOnDiskGraphIndexWriter.Builder` automatically selects the parallel writer at + `build()` time. From 9f555cbacddefdae7dd015012f1b8bd1e0ce749a Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Tue, 21 Jul 2026 18:00:08 -0400 Subject: [PATCH 4/6] adding compression control --- .../jvector/graph/GraphIndexBuilder.java | 34 +++++++- .../jvector/management/CompressionType.java | 33 ++++++++ .../management/GraphIndexBuilderConfig.java | 83 +++++++++++++++++++ .../GraphIndexBuilderConfigMBean.java | 71 ++++++++++++++++ 4 files changed, 220 insertions(+), 1 deletion(-) create mode 100644 jvector-base/src/main/java/io/github/jbellis/jvector/management/CompressionType.java diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/GraphIndexBuilder.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/GraphIndexBuilder.java index d4df1a9bf..9dfa87a3b 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/GraphIndexBuilder.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/GraphIndexBuilder.java @@ -25,7 +25,12 @@ import io.github.jbellis.jvector.graph.similarity.BuildScoreProvider; import io.github.jbellis.jvector.graph.similarity.ScoreFunction; import io.github.jbellis.jvector.graph.similarity.SearchScoreProvider; +import io.github.jbellis.jvector.management.CompressionType; import io.github.jbellis.jvector.management.GraphIndexBuilderConfig; +import io.github.jbellis.jvector.quantization.BinaryQuantization; +import io.github.jbellis.jvector.quantization.BQVectors; +import io.github.jbellis.jvector.quantization.PQVectors; +import io.github.jbellis.jvector.quantization.ProductQuantization; import io.github.jbellis.jvector.util.*; import io.github.jbellis.jvector.vector.VectorSimilarityFunction; import io.github.jbellis.jvector.vector.types.VectorFloat; @@ -109,7 +114,7 @@ public GraphIndexBuilder(RandomAccessVectorValues vectorValues, float neighborOverflow, float alpha) { - this(BuildScoreProvider.randomAccessScoreProvider(vectorValues, similarityFunction), + this(getBuildScoreProvider(vectorValues, similarityFunction), vectorValues.dimension(), M, beamWidth, @@ -117,6 +122,27 @@ public GraphIndexBuilder(RandomAccessVectorValues vectorValues, alpha); } + private static BuildScoreProvider getBuildScoreProvider(RandomAccessVectorValues vectorValues, VectorSimilarityFunction similarityFunction) { + switch(resolveJmxBuildCompressionType()) { + case NONE: + return BuildScoreProvider.randomAccessScoreProvider(vectorValues, similarityFunction); + case PQ: { + var config = GraphIndexBuilderConfig.getInstance(); + int m = vectorValues.dimension() / config.getPqMFactor(); + var compressor = ProductQuantization.compute(vectorValues, m, config.getPqK(), + config.isPqCenterData(), config.getPqAnisotropicThreshold()); + PQVectors pqVectors = compressor.encodeAll(vectorValues, ForkJoinPool.commonPool()); + return BuildScoreProvider.pqBuildScoreProvider(similarityFunction, pqVectors); + } + case BQ: { + BQVectors bqVectors = (BQVectors) BinaryQuantization.compute(vectorValues).encodeAll(vectorValues, ForkJoinPool.commonPool()); + return BuildScoreProvider.bqBuildScoreProvider(bqVectors); + } + default: + throw new IllegalArgumentException("Unsupported build compression type: " + resolveJmxBuildCompressionType()); + } + } + /** * Reads all the vectors from vector values, builds a graph connecting them by their dense * ordinals, using the given hyperparameter settings, and returns the resulting graph. @@ -538,6 +564,12 @@ private static boolean resolveJmxRefineFinalGraph() { return v; } + private static CompressionType resolveJmxBuildCompressionType() { + String v = GraphIndexBuilderConfig.getInstance().getBuildCompressionType(); + logger.debug("buildCompressionType={} (from GraphIndexBuilderConfig)", v); + return CompressionType.valueOf(v); + } + private static boolean logCallerAddHierarchy(boolean v) { logger.debug("addHierarchy={} (caller-provided via deprecated constructor)", v); return v; diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/management/CompressionType.java b/jvector-base/src/main/java/io/github/jbellis/jvector/management/CompressionType.java new file mode 100644 index 000000000..27fbefca4 --- /dev/null +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/management/CompressionType.java @@ -0,0 +1,33 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.github.jbellis.jvector.management; + +public enum CompressionType { + NONE("None"), + PQ("PQ"), + BQ("BQ"); + + private final String type; + + CompressionType(String type) { + this.type = type; + } + + public String getType() { + return type; + } +} diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfig.java b/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfig.java index 832e690ae..a77331c77 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfig.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfig.java @@ -108,6 +108,13 @@ public static GraphIndexBuilderConfig getInstance() { private volatile boolean addHierarchy = true; private volatile boolean refineFinalGraph = true; private volatile boolean parallelBuild = false; + private volatile String buildCompressionType = CompressionType.NONE.name(); + + // PQ build compression parameters — only used when buildCompressionType == "PQ" + private volatile int pqMFactor = 8; + private volatile int pqK = 256; + private volatile boolean pqCenterData = false; + private volatile float pqAnisotropicThreshold = -1.0f; // ── Constructor ────────────────────────────────────────────────────────── @@ -166,4 +173,80 @@ public void setParallelBuild(boolean parallelBuild) { logger.info("JMX: parallelBuild changed {} → {}", previous, parallelBuild); } } + + @Override + public String getBuildCompressionType() { + return this.buildCompressionType; + } + + @Override + public void setBuildCompressionType(String compressionType) { + // Validate eagerly so JMX clients get an error immediately rather than at build time. + CompressionType.valueOf(compressionType); + String previous = this.buildCompressionType; + this.buildCompressionType = compressionType; + if (!previous.equals(compressionType)) { + logger.info("JMX: buildCompressionType changed {} → {}", previous, compressionType); + } + } + + // ── PQ build compression parameters ───────────────────────────────────── + + @Override + public int getPqMFactor() { + return pqMFactor; + } + + @Override + public void setPqMFactor(int mFactor) { + if (mFactor <= 0) throw new IllegalArgumentException("pqMFactor must be positive"); + int previous = this.pqMFactor; + this.pqMFactor = mFactor; + if (previous != mFactor) { + logger.info("JMX: pqMFactor changed {} → {}", previous, mFactor); + } + } + + @Override + public int getPqK() { + return pqK; + } + + @Override + public void setPqK(int k) { + if (k <= 0) throw new IllegalArgumentException("pqK must be positive"); + int previous = this.pqK; + this.pqK = k; + if (previous != k) { + logger.info("JMX: pqK changed {} → {}", previous, k); + } + } + + @Override + public boolean isPqCenterData() { + return pqCenterData; + } + + @Override + public void setPqCenterData(boolean centerData) { + boolean previous = this.pqCenterData; + this.pqCenterData = centerData; + if (previous != centerData) { + logger.info("JMX: pqCenterData changed {} → {}", previous, centerData); + } + } + + @Override + public float getPqAnisotropicThreshold() { + return pqAnisotropicThreshold; + } + + @Override + public void setPqAnisotropicThreshold(float threshold) { + float previous = this.pqAnisotropicThreshold; + this.pqAnisotropicThreshold = threshold; + if (Float.compare(previous, threshold) != 0) { + logger.info("JMX: pqAnisotropicThreshold changed {} → {}", previous, threshold); + } + } } diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfigMBean.java b/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfigMBean.java index 917cd2fee..449bda75d 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfigMBean.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfigMBean.java @@ -91,4 +91,75 @@ public interface GraphIndexBuilderConfigMBean { * @param parallelBuild {@code true} to use the parallel writer, {@code false} for sequential (default) */ void setParallelBuild(boolean parallelBuild); + + /** + * Returns the compression type used during graph construction scoring. + * Valid values are the names of {@link CompressionType} constants: {@code "NONE"}, {@code "PQ"}, {@code "BQ"}. + */ + String getBuildCompressionType(); + + /** + * Sets the compression type used during graph construction scoring. + * + * @param compressionType one of {@code "NONE"}, {@code "PQ"}, {@code "BQ"} + * @throws IllegalArgumentException if the value is not a valid {@link CompressionType} name + */ + void setBuildCompressionType(String compressionType); + + // ── PQ build compression parameters ────────────────────────────────────── + // These are only consulted when BuildCompressionType is "PQ". + + /** + * Returns the PQ subspace divisor. The number of PQ subspaces {@code m} is computed as + * {@code dimension / mFactor}. Ignored when {@code BuildCompressionType} is not {@code "PQ"}. + */ + int getPqMFactor(); + + /** + * Sets the PQ subspace divisor. + * + * @param mFactor must be a positive integer that evenly divides the vector dimension + */ + void setPqMFactor(int mFactor); + + /** + * Returns the number of centroids per PQ subspace (default 256). + * Ignored when {@code BuildCompressionType} is not {@code "PQ"}. + */ + int getPqK(); + + /** + * Sets the number of centroids per PQ subspace. + * + * @param k must be a positive power of two; typical value is 256 + */ + void setPqK(int k); + + /** + * Returns whether PQ training globally centers the data before clustering. + * Recommended {@code true} for Euclidean similarity, {@code false} otherwise. + * Ignored when {@code BuildCompressionType} is not {@code "PQ"}. + */ + boolean isPqCenterData(); + + /** + * Enables or disables global centering during PQ training. + * + * @param centerData {@code true} to center, {@code false} to skip (default) + */ + void setPqCenterData(boolean centerData); + + /** + * Returns the anisotropic loss threshold used during PQ encoding. + * {@code -1.0} disables anisotropic weighting (default). + * Ignored when {@code BuildCompressionType} is not {@code "PQ"}. + */ + float getPqAnisotropicThreshold(); + + /** + * Sets the anisotropic loss threshold. + * + * @param threshold use {@code -1.0} to disable anisotropic weighting (default) + */ + void setPqAnisotropicThreshold(float threshold); } From cae5361c10c41721939d70da3545dbbffe01b50e Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Fri, 24 Jul 2026 15:44:14 -0400 Subject: [PATCH 5/6] tidying up --- .../4.1.0/replace_with_pr.feature.md | 82 +++++++++++++++---- .../jvector/graph/GraphIndexBuilder.java | 5 +- .../management/GraphIndexBuilderConfig.java | 16 +++- .../GraphIndexBuilderConfigMBean.java | 7 +- 4 files changed, 87 insertions(+), 23 deletions(-) diff --git a/docs/release notes/4.1.0/replace_with_pr.feature.md b/docs/release notes/4.1.0/replace_with_pr.feature.md index b924ddc81..f8d1cdf46 100644 --- a/docs/release notes/4.1.0/replace_with_pr.feature.md +++ b/docs/release notes/4.1.0/replace_with_pr.feature.md @@ -5,8 +5,9 @@ Introduces `GraphIndexBuilderConfig`, a JMX-managed singleton that exposes `Grap construction parameters as runtime-tunable attributes. Before this change, options such as `addHierarchy`, `refineFinalGraph`, and `parallelBuild` could only be set at the construction call site and required a code change or application restart to modify. With this change, all -three parameters can be inspected and updated live via any standard JMX client — JConsole, -jvisualvm, jmxterm, or a monitoring agent — without restarting the JVM. +parameters — graph topology flags, write path selection, build-time compression type, and PQ +compression tuning — can be inspected and updated live via any standard JMX client (JConsole, +jvisualvm, jmxterm, or a monitoring agent) without restarting the JVM. Changes take effect the next time a `GraphIndexBuilder` is constructed; they do not affect indexes that are already being built or have already been built. @@ -21,12 +22,34 @@ not disrupt normal operation — the singleton continues to supply its default v **Managed Attributes** +*Graph topology* + | Attribute | Type | Default | Description | |---|---|---|---| | `AddHierarchy` | `boolean` | `true` | When `true`, builds HNSW-style hierarchy layers on top of the base Vamana graph. `false` produces a flat level-0 (plain Vamana) index, which uses less memory and may build faster on small datasets. | | `RefineFinalGraph` | `boolean` | `true` | When `true`, runs a second diversity-refinement pass over each node's edges after the initial build. Improves recall at the cost of additional build time. | + +*Write path* + +| Attribute | Type | Default | Description | +|---|---|---|---| | `ParallelBuild` | `boolean` | `false` | When `true`, serializes level-0 node records concurrently via `OnDiskParallelGraphIndexWriter`. Both writers produce an identical on-disk format; switching this flag does not require re-indexing existing data. | +*Build-time compression* + +| Attribute | Type | Default | Description | +|---|---|---|---| +| `BuildCompressionType` | `String` | `"NONE"` | Compression used for scoring during graph construction. Valid values: `"NONE"` (full-precision), `"PQ"` (Product Quantization), `"BQ"` (Binary Quantization). | + +*PQ build compression parameters* — consulted only when `BuildCompressionType` is `"PQ"` + +| Attribute | Type | Default | Description | +|---|---|---|---| +| `PqMFactor` | `int` | `8` | Subspace divisor: the number of PQ subspaces `m` is computed as `dimension / mFactor`. Must be a positive integer that evenly divides the vector dimension. | +| `PqK` | `int` | `256` | Number of centroids per PQ subspace. Must be positive; conventionally a power of two (256 is the standard value). | +| `PqCenterData` | `boolean` | `false` | When `true`, globally centers the dataset before PQ cluster training. Recommended for Euclidean similarity; typically not needed for cosine or dot-product. | +| `PqAnisotropicThreshold` | `float` | `-1.0` | Anisotropic loss threshold for PQ encoding. `-1.0` disables anisotropic weighting; positive values bias the quantizer toward directions that matter most for inner-product search. | + **How to Enable** `GraphIndexBuilderConfig` is initialized automatically on first access and registers its MBean @@ -41,13 +64,24 @@ import io.github.jbellis.jvector.management.GraphIndexBuilderConfig; GraphIndexBuilderConfig config = GraphIndexBuilderConfig.getInstance(); // Read current values -boolean addHierarchy = config.isAddHierarchy(); -boolean refineFinal = config.isRefineFinalGraph(); -boolean parallelBuild = config.isParallelBuild(); +boolean addHierarchy = config.isAddHierarchy(); +boolean refineFinal = config.isRefineFinalGraph(); +boolean parallelBuild = config.isParallelBuild(); +String buildCompressionType = config.getBuildCompressionType(); // "NONE", "PQ", or "BQ" +int pqMFactor = config.getPqMFactor(); +int pqK = config.getPqK(); +boolean pqCenterData = config.isPqCenterData(); +float pqAnisotropicThreshold = config.getPqAnisotropicThreshold(); // Update at runtime (affects all subsequent GraphIndexBuilder constructions) config.setAddHierarchy(false); config.setParallelBuild(true); + +// Switch to PQ build-time compression with custom parameters +config.setBuildCompressionType("PQ"); +config.setPqMFactor(4); // dimension / 4 subspaces +config.setPqK(256); +config.setPqCenterData(true); // recommended for Euclidean similarity ``` *Non-deprecated constructor* — `GraphIndexBuilder` constructors that do not accept explicit @@ -95,13 +129,18 @@ JConsole is the standard JMX browser included with every JDK installation. 3. **Read an attribute** - Click on **Attributes**. The right-hand panel lists all three attributes with their + Click on **Attributes**. The right-hand panel lists all attributes with their current values: ``` - AddHierarchy true - RefineFinalGraph true - ParallelBuild false + AddHierarchy true + RefineFinalGraph true + ParallelBuild false + BuildCompressionType NONE + PqMFactor 8 + PqK 256 + PqCenterData false + PqAnisotropicThreshold -1.0 ``` 4. **Set an attribute** @@ -131,10 +170,18 @@ info -b # Read a specific attribute get AddHierarchy +get BuildCompressionType -# Set an attribute +# Set graph topology flags set AddHierarchy false set ParallelBuild true + +# Switch to PQ build-time compression +set BuildCompressionType PQ +set PqMFactor 4 +set PqK 256 +set PqCenterData true +set PqAnisotropicThreshold -1.0 ``` **Notes** @@ -145,7 +192,14 @@ set ParallelBuild true after the change in the same JVM. Per-graph overrides are not supported in this release; callers that need per-graph control should pass values explicitly to the appropriate constructor. -- The `parallelBuild` attribute requires the `OnDiskParallelGraphIndexWriter` to be on the - classpath (it is part of `jvector-base`). When `parallelBuild` is `true`, the unified - `RandomAccessOnDiskGraphIndexWriter.Builder` automatically selects the parallel writer at - `build()` time. +- The `parallelBuild` attribute requires `OnDiskParallelGraphIndexWriter` to be on the + classpath (it is part of `jvector-base`). Both writers produce an identical on-disk format; + switching this flag does not require re-indexing existing data. +- The PQ parameters (`PqMFactor`, `PqK`, `PqCenterData`, `PqAnisotropicThreshold`) are only + consulted when `BuildCompressionType` is `"PQ"`. Setting them while `BuildCompressionType` + is `"NONE"` or `"BQ"` has no effect on the current build but the values are retained and + will apply if `BuildCompressionType` is later changed to `"PQ"`. +- `setBuildCompressionType` accepts values case-insensitively (`"none"`, `"None"`, and `"NONE"` + are all valid) and normalizes to the canonical name on storage. It validates immediately and + throws `IllegalArgumentException` for unrecognised strings, so JMX clients receive an error + at set time rather than silently at build time. diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/GraphIndexBuilder.java b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/GraphIndexBuilder.java index 9dfa87a3b..5f733f559 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/graph/GraphIndexBuilder.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/graph/GraphIndexBuilder.java @@ -123,7 +123,8 @@ public GraphIndexBuilder(RandomAccessVectorValues vectorValues, } private static BuildScoreProvider getBuildScoreProvider(RandomAccessVectorValues vectorValues, VectorSimilarityFunction similarityFunction) { - switch(resolveJmxBuildCompressionType()) { + CompressionType type = resolveJmxBuildCompressionType(); + switch(type) { case NONE: return BuildScoreProvider.randomAccessScoreProvider(vectorValues, similarityFunction); case PQ: { @@ -139,7 +140,7 @@ private static BuildScoreProvider getBuildScoreProvider(RandomAccessVectorValues return BuildScoreProvider.bqBuildScoreProvider(bqVectors); } default: - throw new IllegalArgumentException("Unsupported build compression type: " + resolveJmxBuildCompressionType()); + throw new IllegalArgumentException("Unsupported build compression type: " + type); } } diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfig.java b/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfig.java index a77331c77..26eb3a7d5 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfig.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfig.java @@ -23,6 +23,7 @@ import javax.management.MBeanServer; import javax.management.ObjectName; import java.lang.management.ManagementFactory; +import java.util.Locale; /** * Singleton that holds JMX-managed default values for @@ -182,11 +183,18 @@ public String getBuildCompressionType() { @Override public void setBuildCompressionType(String compressionType) { // Validate eagerly so JMX clients get an error immediately rather than at build time. - CompressionType.valueOf(compressionType); + // Matching is case-insensitive; the canonical enum name is stored for consistency. + CompressionType ct; + try { + ct = CompressionType.valueOf(compressionType.toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid build compression type: '" + compressionType + "'. Valid values: NONE, PQ, BQ", e); + } + String canonical = ct.name(); String previous = this.buildCompressionType; - this.buildCompressionType = compressionType; - if (!previous.equals(compressionType)) { - logger.info("JMX: buildCompressionType changed {} → {}", previous, compressionType); + this.buildCompressionType = canonical; + if (!previous.equals(canonical)) { + logger.info("JMX: buildCompressionType changed {} → {}", previous, canonical); } } diff --git a/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfigMBean.java b/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfigMBean.java index 449bda75d..42a930dc1 100644 --- a/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfigMBean.java +++ b/jvector-base/src/main/java/io/github/jbellis/jvector/management/GraphIndexBuilderConfigMBean.java @@ -100,9 +100,10 @@ public interface GraphIndexBuilderConfigMBean { /** * Sets the compression type used during graph construction scoring. + * Matching is case-insensitive; the value is normalized to the canonical enum name on storage. * - * @param compressionType one of {@code "NONE"}, {@code "PQ"}, {@code "BQ"} - * @throws IllegalArgumentException if the value is not a valid {@link CompressionType} name + * @param compressionType one of {@code "NONE"}, {@code "PQ"}, {@code "BQ"} (case-insensitive) + * @throws IllegalArgumentException if the value does not match any {@link CompressionType} */ void setBuildCompressionType(String compressionType); @@ -131,7 +132,7 @@ public interface GraphIndexBuilderConfigMBean { /** * Sets the number of centroids per PQ subspace. * - * @param k must be a positive power of two; typical value is 256 + * @param k must be positive; conventionally a power of two (typical value 256) */ void setPqK(int k); From bba389d3705ace1f230f6d019da0ec9cde9b584e Mon Sep 17 00:00:00 2001 From: Mark Wolters Date: Fri, 24 Jul 2026 16:12:11 -0400 Subject: [PATCH 6/6] rename release notes to pr number --- .../4.1.0/{replace_with_pr.feature.md => 703.feature.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/release notes/4.1.0/{replace_with_pr.feature.md => 703.feature.md} (100%) diff --git a/docs/release notes/4.1.0/replace_with_pr.feature.md b/docs/release notes/4.1.0/703.feature.md similarity index 100% rename from docs/release notes/4.1.0/replace_with_pr.feature.md rename to docs/release notes/4.1.0/703.feature.md