Bug/acquisition maximizer sampling (#1106)

* Adapt how n_points works in the config_selector.py

Also remove LocalAndSortedPriorRandomSearch by integrating it into LocalAndSortedRandomSearch

* Change retrain_after to 1 for BlackBOx Facade

* Fix bugs in local_and_random_search.py

* Edit CHANGELOG.md

* fix acq maximizer docstring

* call get config selector from super class

* Update random and local search to new functionality

* Update CHANGELOG.md

* Fix sampling of initial points for local search when less previous configs than n_points

* Re-format blackbox_facade.py

---------

Co-authored-by: Helena Graf <[email protected]>
Co-authored-by: dengdifan <[email protected]>

CHANGELOG.md

@@ -1,4 +1,11 @@
-# 2.0.3
+# 2.1.0
+
+## Improvements
+- Change the surrogate model to be retrained after every iteration by default in the case of blackbox optimization
+  (#1106).
+- Integrate `LocalAndSortedPriorRandomSearch` functionality into `LocalAndSortedRandomSearch` (#1106).
+- Change the way the `LocalAndSortedRandomSearch` works such that the incumbent always is a starting point and that
+      random configurations are sampled as the basis of the local search, not in addition (#1106).
 
 ## Bugfixes
 - Fix path for dask scheduler file (#1055).
@@ -7,6 +14,10 @@
 - Propagate the Scenario random seed to `get_random_design` (#1066).
 - Configurations that fail to become incumbents will be added to the rejected lists (#1069).
 - SMAC RandomForest doesn't crash when `np.integer` used, i.e. as generated from a `np.random.RandomState` (#1084).
+- Fix the handling of n_points/ challengers in the acquisition maximizers, such that this number now functions as the
+     number of points that are sampled from the acquisition function to find the next challengers. Now also doesn't
+     restrict the config selector to n_retrain many points for finding the max, and instead uses the defaults that are
+     defined via facades/ scenarios (#1106).
 
 ## Misc
 - ci: Update action version (#1072).

smac/acquisition/maximizer/__init__.py

@@ -3,7 +3,6 @@
 )
 from smac.acquisition.maximizer.differential_evolution import DifferentialEvolution
 from smac.acquisition.maximizer.local_and_random_search import (
-    LocalAndSortedPriorRandomSearch,
     LocalAndSortedRandomSearch,
 )
 from smac.acquisition.maximizer.local_search import LocalSearch
@@ -13,7 +12,6 @@
     "AbstractAcquisitionMaximizer",
     "DifferentialEvolution",
     "LocalAndSortedRandomSearch",
-    "LocalAndSortedPriorRandomSearch",
     "LocalSearch",
     "RandomSearch",
 ]

smac/acquisition/maximizer/abstract_acqusition_maximizer.py

@@ -27,12 +27,9 @@ class AbstractAcquisitionMaximizer:
 
     Parameters
     ----------
-    configspace : ConfigurationSpace
-    acquisition_function : AbstractAcquisitionFunction
-    challengers : int, defaults to 5000
-        Number of configurations to sample from the configuration space to get
-        the acquisition function value for, thus challenging the current
-        incumbent and becoming a candidate for the next function evaluation.
+    configspace : ConfigurationSpace acquisition_function : AbstractAcquisitionFunction
+    challengers : int, defaults to 5000 Number of configurations sampled during the optimization process,
+    details depend on the used maximizer. Also, the number of configurations that is returned by calling `maximize`.
     seed : int, defaults to 0
     """
 
@@ -85,8 +82,8 @@ def maximize(
         previous_configs: list[Configuration]
             Previous evaluated configurations.
         n_points: int, defaults to None
-            Number of points to be sampled. If `n_points` is not specified,
-            `self._challengers` is used.
+            Number of points to be sampled & number of configurations to be returned. If `n_points` is not specified,
+            `self._challengers` is used. Semantics depend on concrete implementation.
         random_design: AbstractRandomDesign, defaults to None
             Part of the returned ChallengerList such that we can interleave random configurations
             by a scheme defined by the random design. The method `random_design.next_iteration()`

smac/acquisition/maximizer/differential_evolution.py

@@ -30,6 +30,7 @@ def _maximize(
         previous_configs: list[Configuration],
         n_points: int,
     ) -> list[tuple[float, Configuration]]:
+        # n_points is not used here, but is required by the interface
 
         configs: list[tuple[float, Configuration]] = []
 

smac/acquisition/maximizer/helpers.py

@@ -4,8 +4,8 @@
 
 from ConfigSpace import Configuration, ConfigurationSpace
 
+from smac.random_design import ProbabilityRandomDesign
 from smac.random_design.abstract_random_design import AbstractRandomDesign
-from smac.random_design.modulus_design import ModulusRandomDesign
 
 
 class ChallengerList(Iterator):
@@ -20,7 +20,7 @@ class ChallengerList(Iterator):
     ----------
     configspace : ConfigurationSpace
     challenger_callback : Callable
-        Callback function which returns a list of challengers (without interleaved random configurations, must a be a
+        Callback function which returns a list of challengers (without interleaved random configurations), must a be a
         python closure.
     random_design : AbstractRandomDesign | None, defaults to ModulusRandomDesign(modulus=2.0)
         Which random design should be used.
@@ -30,7 +30,7 @@ def __init__(
         self,
         configspace: ConfigurationSpace,
         challenger_callback: Callable,
-        random_design: AbstractRandomDesign | None = ModulusRandomDesign(modulus=2.0),
+        random_design: AbstractRandomDesign | None = ProbabilityRandomDesign(seed=0, probability=0.08447232371720552),
     ):
         self._challengers_callback = challenger_callback
         self._challengers: list[Configuration] | None = None
@@ -72,44 +72,3 @@ def __len__(self) -> int:
             self._challengers = self._challengers_callback()
 
         return len(self._challengers) - self._index
-
-
-'''
-class FixedSet(AbstractAcquisitionMaximizer):
-    def __init__(
-        self,
-        configurations: list[Configuration],
-        acquisition_function: AbstractAcquisitionFunction,
-        configspace: ConfigurationSpace,
-        challengers: int = 5000,
-        seed: int = 0,
-    ):
-        """Maximize the acquisition function over a finite list of configurations.
-
-        Parameters
-        ----------
-        configurations : list[~smac._configspace.Configuration]
-            Candidate configurations
-        acquisition_function : ~smac.acquisition.AbstractAcquisitionFunction
-
-        configspace : ~smac._configspace.ConfigurationSpace
-
-        rng : np.random.RandomState or int, optional
-        """
-        super().__init__(
-            acquisition_function=acquisition_function, configspace=configspace, challengers=challengers, seed=seed
-        )
-        self.configurations = configurations
-
-    def _maximize(
-        self,
-        runhistory: RunHistory,
-        stats: Stats,
-        n_points: int,
-    ) -> list[tuple[float, Configuration]]:
-        configurations = copy.deepcopy(self.configurations)
-        for config in configurations:
-            config.origin = "Fixed Set"
-
-        return self._sort_by_acquisition_value(configurations)
-'''