0.35.2

Author: bartzbeielstein

notebooks/spot_rosenbrock_6d.ipynb

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "spotpython"
-version = "0.35.1"
+version = "0.35.2"
 authors = [
   { name="T. Bartz-Beielstein", email="tbb@bartzundbartz.de" }
 ]

pyproject.toml

@@ -36,7 +36,7 @@
 from spotpython.utils.convert import get_shape
 from spotpython.utils.init import fun_control_init, optimizer_control_init, surrogate_control_init, design_control_init
 from spotpython.utils.compare import selectNew
-from spotpython.utils.aggregate import aggregate_mean_var, select_distant_points
+from spotpython.utils.aggregate import aggregate_mean_var, select_distant_points, select_best_cluster
 from spotpython.utils.repair import remove_nan, repair_non_numeric
 from spotpython.utils.file import get_experiment_filename, get_result_filename
 from spotpython.budget.ocba import get_ocba_X
@@ -215,10 +215,6 @@ def __init__(
         # Kernel selection from fun_control (NEW)
         self.kernel = None
         self.kernel_params = None
-        if fun_control is not None:
-            self.kernel = fun_control.get("kernel", "gauss")
-            self.kernel_params = fun_control.get("kernel_params", {})
-
         self.counter = 0
         self.success_rate = 0.0
         self.success_counter = 0
@@ -227,6 +223,7 @@ def __init__(
         # small value:
         self.eps = sqrt(spacing(1))
 
+        self.selection_method = "distant"  # or "best"
         self._set_fun(fun)
 
         self._set_bounds_and_dim()
@@ -449,11 +446,16 @@ def _set_additional_attributes(self) -> None:
         self.progress_file = self.fun_control["progress_file"]
         self.tkagg = self.fun_control["tkagg"]
         self.min_success_rate = self.fun_control["min_success_rate"]
+        self.selection_method = self.fun_control["selection_method"]
         # self.success_counter = 0
         if self.tkagg:
             matplotlib.use("TkAgg")
         self.verbosity = self.fun_control["verbosity"]
         self.acquisition_failure_strategy = self.fun_control["acquisition_failure_strategy"]
+        self.kernel = self.fun_control["kernel"]
+        self.kernel_params = self.fun_control["kernel_params"]
+
+        # Surrogate control attributes:
         self.max_surrogate_points = self.surrogate_control["max_surrogate_points"]
         self.use_nystrom = self.surrogate_control["use_nystrom"]
         self.nystrom_m = self.surrogate_control["nystrom_m"]
@@ -1640,6 +1642,57 @@ def update_stats(self) -> None:
             # variance of the best mean y value so far:
             self.min_var_y = self.var_y[argmin(self.mean_y)]
 
+    def selection_dispatcher(self) -> Tuple[np.ndarray, np.ndarray]:
+        """
+        Dispatcher for selection methods.
+        Depending on the value of `self.selection_method`,
+        it calls the appropriate selection function.
+        Args:
+            self (object): Spot object
+        Returns:
+            Tuple[numpy.ndarray, numpy.ndarray]:
+                selected design points and their corresponding function values
+        Attributes:
+            self.selection_method (str):
+                selection method to use
+            self.X (numpy.ndarray):
+                design points
+            self.y (numpy.ndarray):
+                function values
+            self.max_surrogate_points (int):
+                maximum number of points to select
+        Examples:
+            >>> import numpy as np
+                from spotpython.fun.objectivefunctions import Analytical
+                from spotpython.spot import spot
+                from spotpython.utils.init import (
+                    fun_control_init, optimizer_control_init, surrogate_control_init, design_control_init
+                    )
+                # number of initial points:
+                ni = 0
+                X_start = np.array([[0, 0], [0, 1], [   1, 0], [1, 1], [1, 1]])
+                fun = Analytical().fun_sphere
+                fun_control = fun_control_init(
+                    lower = np.array([-1, -1]),
+                    upper = np.array([1, 1])
+                    )
+                design_control=design_control_init(init_size=ni)
+                S = spot.Spot(fun=fun,
+                            fun_control=fun_control,
+                            design_control=design_control,)
+                S.initialize_design(X_start=X_start)
+                S.update_stats()
+                X_S, y_S = S.selection_dispatcher()
+                print(f"Selected X_S: {X_S}")
+                print(f"Selected y_S: {y_S}")
+        """
+        if self.selection_method == "distant":
+            return select_distant_points(X=self.X, y=self.y, k=self.max_surrogate_points)
+        elif self.selection_method == "best":
+            return select_best_cluster(X=self.X, y=self.y, k=self.max_surrogate_points)
+        # If no selection is needed, return all points:
+        return self.X, self.y
+
     def fit_surrogate(self) -> None:
         """
         Fit surrogate model. The surrogate model
@@ -1705,15 +1758,15 @@ def fit_surrogate(self) -> None:
         logger.debug("In fit_surrogate(): self.X.shape: %s", self.X.shape)
         logger.debug("In fit_surrogate(): self.y.shape: %s", self.y.shape)
         # Pass kernel options to surrogate if Kriging is used
-        if hasattr(self.surrogate, "kernel"):
+        if hasattr(self.surrogate, "kernel") and isinstance(self.surrogate, Kriging):
             self.surrogate.kernel = self.kernel
             self.surrogate.kernel_params = self.kernel_params
         X_points = self.X.shape[0]
         y_points = self.y.shape[0]
         if X_points == y_points:
             if (X_points > self.max_surrogate_points) and (self.use_nystrom is False):
-                logger.info("Selecting distant points for surrogate fitting.")
-                X_S, y_S = select_distant_points(X=self.X, y=self.y, k=self.max_surrogate_points)
+                logger.info("Selecting subset of points for surrogate fitting.")
+                X_S, y_S = self.selection_dispatcher()
             else:
                 X_S = self.X
                 y_S = self.y

src/spotpython/spot/spot.py

@@ -296,3 +296,41 @@ def select_distant_points(X, y, k):
     # Select the corresponding y values
     selected_y = y[indices]
     return selected_points, selected_y
+
+
+def select_best_cluster(X, y, k):
+    """
+    Selects all points from the cluster whose center has the smallest mean y value.
+
+    Args:
+        X (numpy.ndarray): X array, shape `(n, k)`.
+        y (numpy.ndarray): values, shape `(n,)`.
+        k (int): number of clusters.
+
+    Returns:
+        (numpy.ndarray): selected `X` values from the best cluster, shape `(m, k)`.
+        (numpy.ndarray): selected `y` values from the best cluster, shape `(m,)`.
+
+    Examples:
+        >>> X = np.array([[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]])
+        >>> y = np.array([5, 4, 3, 2, 1])
+        >>> X_best, y_best = select_best_cluster(X, y, 2)
+        >>> print(X_best)
+        >>> print(y_best)
+    """
+    # Perform k-means clustering
+    kmeans = KMeans(n_clusters=k, random_state=0, n_init="auto").fit(X)
+    labels = kmeans.labels_
+    # Compute mean y for each cluster
+    cluster_means = []
+    for cluster_idx in range(k):
+        cluster_y = y[labels == cluster_idx]
+        if len(cluster_y) == 0:
+            cluster_means.append(np.inf)
+        else:
+            cluster_means.append(np.mean(cluster_y))
+    # Find cluster with smallest mean y
+    best_cluster = np.argmin(cluster_means)
+    # Select all points from the best cluster
+    mask = labels == best_cluster
+    return X[mask], y[mask]

src/spotpython/utils/aggregate.py

@@ -88,6 +88,7 @@ def fun_control_init(
     scaler_name=None,
     scenario=None,
     seed=123,
+    selection_method="distant",
     show_config=False,
     show_models=False,
     show_progress=True,
@@ -291,6 +292,10 @@ def fun_control_init(
             The scenario to use. Default is None. Can be "river", "sklearn", or "lightning".
         seed (int):
             The seed to use for the random number generator. Default is 123.
+        selection_method (str):
+            The method to select points for surrogate fitting when the number of points
+            exceeds max_surrogate_points. Can be either "distant" or "best".
+            Default is "distant".
         sigma (float):
             The standard deviation of the noise of the objective function.
         show_progress (bool):
@@ -423,6 +428,7 @@ def fun_control_init(
                 'prep_model_name': None,
                 'scenario': "lightning",
                 'seed': 1234,
+                'selection_method': "distant",
                 'show_batch_interval': 1000000,
                 'shuffle': None,
                 'sigma': 0.0,
@@ -530,6 +536,7 @@ def fun_control_init(
         "scaler_name": scaler_name,
         "scenario": scenario,
         "seed": seed,
+        "selection_method": selection_method,
         "show_batch_interval": 1_000_000,
         "show_config": show_config,
         "show_models": show_models,

src/spotpython/utils/init.py

@@ -73,6 +73,7 @@ def test_get_spot_attributes_as_df():
                             'progress_file',
                             'red_dim',
                             'rng',
+                            'selection_method',
                             'show_models',
                             'show_progress',
                             'spot_writer',

test/test_get_spot_attributes_as_df.py

@@ -0,0 +1,42 @@
+import numpy as np
+from math import inf
+from spotpython.fun.objectivefunctions import Analytical
+from spotpython.spot import Spot
+from spotpython.surrogate.kriging import Kriging
+fun = Analytical().fun_branin
+from spotpython.utils.init import fun_control_init, design_control_init
+# Needed for the sklearn surrogates:
+from sklearn.gaussian_process import GaussianProcessRegressor
+from sklearn.gaussian_process.kernels import RBF
+from sklearn.tree import DecisionTreeRegressor
+from sklearn.ensemble import RandomForestRegressor
+from sklearn import linear_model
+
+
+PREFIX = "04"
+fun_control = fun_control_init(
+    PREFIX=PREFIX,
+    lower = np.array([-5,-0]),
+    upper = np.array([10,15]),
+    fun_evals=10,
+    max_time=1)
+
+design_control = design_control_init(
+    init_size=10)
+
+S_0 = Kriging(name='kriging', seed=123)
+
+
+def test_spot_2_GP():
+    kernel = 1 * RBF(length_scale=1.0, length_scale_bounds=(1e-2, 1e2))
+    S_GP = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=9)
+    isinstance(S_GP, GaussianProcessRegressor) 
+    isinstance(S_0, Kriging)
+    fun = Analytical(seed=123).fun_branin
+    spot_2_GP = Spot(fun=fun,
+                        fun_control=fun_control,
+                        design_control=design_control,
+                        surrogate = S_GP)
+    spot_2_GP.run()
+    success_GP = True
+    assert success_GP is True

test/test_spot_2.py

@@ -0,0 +1,56 @@
+import numpy as np
+from spotpython.fun.objectivefunctions import Analytical
+from spotpython.spot import Spot
+from spotpython.utils.init import fun_control_init, design_control_init, surrogate_control_init
+
+dim = 2
+lower = np.full(dim, -2)
+upper = np.full(dim, 2)
+fun = Analytical().fun_rosenbrock
+fun_evals = 10
+
+init_size = 5
+use_nystrom = False
+method = "regression"
+infill_criterion = "y"
+tolerance_x = 1e-9
+seed = 321
+max_surrogate_points = 7
+min_Lambda = -4
+max_Lambda = 3
+min_theta = -3
+max_theta = 2
+isotropic = False
+kernel = "matern"
+kernel_params = {"nu": 1.5}
+selection_method = "distance"
+min_success_rate = 0.2
+
+fun_control = fun_control_init(
+    lower=lower,
+    upper=upper,
+    fun_evals=fun_evals,
+    seed=seed,
+    show_progress=True,
+    infill_criterion=infill_criterion,
+    tolerance_x=tolerance_x,
+    TENSORBOARD_CLEAN=True,
+    tensorboard_log=True,
+    kernel=kernel,
+    kernel_params=kernel_params,
+    selection_method=selection_method,
+    min_success_rate=min_success_rate
+)
+design_control = design_control_init(init_size=init_size)
+surrogate_control_exact = surrogate_control_init(use_nystrom=use_nystrom, method=method, max_surrogate_points=max_surrogate_points, min_Lambda=min_Lambda, max_Lambda=max_Lambda, min_theta=min_theta, max_theta=max_theta, isotropic=isotropic)
+
+def test_run_1():
+    spot_exact_y = Spot(
+        fun=fun,
+        fun_control=fun_control,
+        design_control=design_control,
+        surrogate_control=surrogate_control_exact
+    )
+    spot_exact_y.run()
+    exact_success_y = True
+    assert exact_success_y is True

test/test_spot_run_1.py

@@ -5,7 +5,7 @@
 @pytest.fixture
 def spot_instance():
     # Minimal Spot instance with required attributes
-    fun_control = {"lower": np.array([-1, -1]), "upper": np.array([1, 1]), "var_type": ["num", "num"], "fun_evals": 10, "fun_repeats": 1, "max_time": 1e6, "noise": False, "tolerance_x": 0, "ocba_delta": 0, "log_level": 50, "show_models": False, "show_progress": False, "infill_criterion": "ei", "n_points": 1, "seed": 1, "progress_file": None, "tkagg": False, "verbosity": 0, "acquisition_failure_strategy": "random", "fun_mo2so": None, "penalty_NA": 0, "PREFIX": "test", "tensorboard_log": False, "spot_tensorboard_path": None, "save_experiment": False, "db_dict_name": None, "save_result": False, "var_name": ["x0", "x1"], "min_success_rate": 0.0}
+    fun_control = {"lower": np.array([-1, -1]), "upper": np.array([1, 1]), "var_type": ["num", "num"], "fun_evals": 10, "fun_repeats": 1, "max_time": 1e6, "noise": False, "tolerance_x": 0, "ocba_delta": 0, "log_level": 50, "show_models": False, "show_progress": False, "infill_criterion": "ei", "n_points": 1, "seed": 1, "progress_file": None, "tkagg": False, "verbosity": 0, "acquisition_failure_strategy": "random", "fun_mo2so": None, "penalty_NA": 0, "PREFIX": "test", "tensorboard_log": False, "spot_tensorboard_path": None, "save_experiment": False, "db_dict_name": None, "save_result": False, "var_name": ["x0", "x1"], "min_success_rate": 0.0, "selection_method": "distant", "kernel": "gauss", "kernel_params": {}}
     design_control = {"init_size": 5, "repeats": 1}
     from spotpython.utils.init import surrogate_control_init, optimizer_control_init
     surrogate_control = surrogate_control_init()