diff --git a/CHANGELOG.md b/CHANGELOG.md index 850814871..0b63c2816 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `CustomDiscreteParameter` does not allow duplicated rows in `data` anymore - De-/activating Polars via `BAYBE_DEACTIVATE_POLARS` now requires passing values compatible with `strtobool` +- `allow_repeated_recommendations` has been renamed to + `allow_recommending_already_recommended` ### Fixed - Rare bug arising from degenerate `SubstanceParameter.comp_df` rows that caused @@ -30,6 +32,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Search spaces are now stateless, preventing unintended side effects that could lead to incorrect candidate sets when reused in different optimization contexts - `qNIPV` not working with single `MIN` targets +- `allow_*` flags are now context-aware, i.e. setting them in a context where they are + irrelevant now raises an error instead of passing silently ### Deprecations - Passing a dataframe via the `data` argument to `Objective.transform` is no longer @@ -44,6 +48,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `SubstanceEncoding` value `RDKIT`. As a replacement, `RDKIT2DDESCRIPTORS` can be used. - The `metadata` attribute of `SubspaceDiscrete` no longer exists. Metadata is now exclusively handled by the `Campaign` class. +- Passing `allow_*` flags to recommenders is no longer supported since the necessary + metadata required for the flags is no longer available at that level. The + functionality has been taken over by `Campaign`. ## [0.11.3] - 2024-11-06 ### Fixed diff --git a/baybe/campaign.py b/baybe/campaign.py index 6bd36585b..92c341a65 100644 --- a/baybe/campaign.py +++ b/baybe/campaign.py @@ -4,27 +4,27 @@ import gc import json -from collections.abc import Collection +from collections.abc import Callable, Collection from functools import reduce -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import cattrs import numpy as np import pandas as pd -from attrs import define, evolve, field +from attrs import Attribute, Factory, define, evolve, field, fields from attrs.converters import optional from attrs.validators import instance_of from typing_extensions import override from baybe.constraints.base import DiscreteConstraint -from baybe.exceptions import IncompatibilityError +from baybe.exceptions import IncompatibilityError, NotEnoughPointsLeftError from baybe.objectives.base import Objective, to_objective from baybe.parameters.base import Parameter from baybe.recommenders.base import RecommenderProtocol from baybe.recommenders.meta.base import MetaRecommender from baybe.recommenders.meta.sequential import TwoPhaseMetaRecommender from baybe.recommenders.pure.bayesian.base import BayesianRecommender -from baybe.searchspace._annotated import AnnotatedSubspaceDiscrete +from baybe.searchspace._filtered import FilteredSubspaceDiscrete from baybe.searchspace.core import ( SearchSpace, SearchSpaceType, @@ -39,7 +39,7 @@ telemetry_record_recommended_measurement_percentage, telemetry_record_value, ) -from baybe.utils.basic import is_all_instance +from baybe.utils.basic import UNSPECIFIED, UnspecifiedType, is_all_instance from baybe.utils.boolean import eq_dataframe from baybe.utils.dataframe import filter_df, fuzzy_row_match from baybe.utils.plotting import to_string @@ -54,6 +54,38 @@ _METADATA_COLUMNS = [_RECOMMENDED, _MEASURED, _EXCLUDED] +def _make_allow_flag_default_factory( + default: bool, +) -> Callable[[Campaign], bool | UnspecifiedType]: + """Make a default factory for allow_* flags.""" + + def default_allow_flag(campaign: Campaign) -> bool | UnspecifiedType: + """Attrs-compatible default factory for allow_* flags.""" + if campaign.searchspace.type is SearchSpaceType.DISCRETE: + return default + return UNSPECIFIED + + return default_allow_flag + + +def _validate_allow_flag(campaign: Campaign, attribute: Attribute, value: Any) -> None: + """Attrs-compatible validator for context-aware validation of allow_* flags.""" + match campaign.searchspace.type: + case SearchSpaceType.DISCRETE: + if not isinstance(value, bool): + raise ValueError( + f"For search spaces of '{SearchSpaceType.DISCRETE}', " + f"'{attribute.name}' must be a Boolean." + ) + case _: + if value is not UNSPECIFIED: + raise ValueError( + f"For search spaces of type other than " + f"'{SearchSpaceType.DISCRETE}', '{attribute.name}' cannot be set " + f"since the flag is meaningless in such contexts.", + ) + + @define class Campaign(SerialMixin): """Main class for interaction with BayBE. @@ -88,6 +120,36 @@ class Campaign(SerialMixin): ) """The employed recommender""" + allow_recommending_already_measured: bool | UnspecifiedType = field( + default=Factory( + _make_allow_flag_default_factory(default=True), takes_self=True + ), + validator=_validate_allow_flag, + kw_only=True, + ) + """Allow to recommend experiments that were already measured earlier. + Can only be set for discrete search spaces.""" + + allow_recommending_already_recommended: bool | UnspecifiedType = field( + default=Factory( + _make_allow_flag_default_factory(default=False), takes_self=True + ), + validator=_validate_allow_flag, + kw_only=True, + ) + """Allow to recommend experiments that were already recommended earlier. + Can only be set for discrete search spaces.""" + + allow_recommending_pending_experiments: bool | UnspecifiedType = field( + default=Factory( + _make_allow_flag_default_factory(default=False), takes_self=True + ), + validator=_validate_allow_flag, + kw_only=True, + ) + """Allow pending experiments to be part of the recommendations. + Can only be set for discrete search spaces.""" + # Metadata _searchspace_metadata: pd.DataFrame = field(init=False, eq=eq_dataframe) """Metadata tracking the experimentation status of the search space.""" @@ -371,21 +433,71 @@ def recommend( self._measurements_exp.fillna({"FitNr": self.n_fits_done}, inplace=True) # Prepare the search space according to the current campaign state - annotated_searchspace = evolve( - self.searchspace, - discrete=AnnotatedSubspaceDiscrete.from_subspace( - self.searchspace.discrete, self._searchspace_metadata - ), - ) + if self.searchspace.type is SearchSpaceType.DISCRETE: + # TODO: This implementation should at some point be hidden behind an + # appropriate public interface, like `SubspaceDiscrete.filter()` + mask_todrop = self._searchspace_metadata[_EXCLUDED].copy() + if not self.allow_recommending_already_recommended: + mask_todrop |= self._searchspace_metadata[_RECOMMENDED] + if not self.allow_recommending_already_measured: + mask_todrop |= self._searchspace_metadata[_MEASURED] + if ( + not self.allow_recommending_pending_experiments + and pending_experiments is not None + ): + mask_todrop |= pd.merge( + self.searchspace.discrete.exp_rep, + pending_experiments, + indicator=True, + how="left", + )["_merge"].eq("both") + searchspace = evolve( + self.searchspace, + discrete=FilteredSubspaceDiscrete.from_subspace( + self.searchspace.discrete, ~mask_todrop.to_numpy() + ), + ) + else: + searchspace = self.searchspace # Get the recommended search space entries - rec = self.recommender.recommend( - batch_size, - annotated_searchspace, - self.objective, - self._measurements_exp, - pending_experiments, - ) + try: + rec = self.recommender.recommend( + batch_size, + searchspace, + self.objective, + self._measurements_exp, + pending_experiments, + ) + except NotEnoughPointsLeftError as ex: + # Aliases for code compactness + f = fields(Campaign) + ok_m = self.allow_recommending_already_measured + ok_r = self.allow_recommending_already_recommended + ok_p = self.allow_recommending_pending_experiments + ok_m_name = f.allow_recommending_already_measured.name + ok_r_name = f.allow_recommending_already_recommended.name + ok_p_name = f.allow_recommending_pending_experiments.name + no_blocked_pending_points = ok_p or (pending_experiments is None) + + # If there are no candidate restrictions to be relaxed + if ok_m and ok_r and no_blocked_pending_points: + raise ex + + # Otherwise, extract possible relaxations + solution = [ + f"'{name}=True'" + for name, value in [ + (ok_m_name, ok_m), + (ok_r_name, ok_r), + (ok_p_name, no_blocked_pending_points), + ] + if not value + ] + message = solution[0] if len(solution) == 1 else " and/or ".join(solution) + raise NotEnoughPointsLeftError( + f"{str(ex)} Consider setting {message}." + ) from ex # Cache the recommendations self._cached_recommendation = rec.copy() diff --git a/baybe/recommenders/base.py b/baybe/recommenders/base.py index 5f8f4724c..1f9ec3d45 100644 --- a/baybe/recommenders/base.py +++ b/baybe/recommenders/base.py @@ -68,6 +68,11 @@ def recommend( _surrogate_model=cattrs.override(rename="surrogate_model"), _current_recommender=cattrs.override(omit=False), _used_recommender_ids=cattrs.override(omit=False), + _deprecated_allow_repeated_recommendations=cattrs.override(omit=True), + _deprecated_allow_recommending_already_measured=cattrs.override(omit=True), + _deprecated_allow_recommending_pending_experiments=cattrs.override( + omit=True + ), ), ), ) diff --git a/baybe/recommenders/meta/base.py b/baybe/recommenders/meta/base.py index 6c39c11db..1d08be54a 100644 --- a/baybe/recommenders/meta/base.py +++ b/baybe/recommenders/meta/base.py @@ -4,7 +4,6 @@ from abc import ABC, abstractmethod from typing import Any -import cattrs import pandas as pd from attrs import define, field from typing_extensions import override @@ -135,17 +134,7 @@ def recommend( # Register (un-)structure hooks -converter.register_unstructure_hook( - MetaRecommender, - lambda x: unstructure_base( - x, - # TODO: Remove once deprecation got expired: - overrides=dict( - allow_repeated_recommendations=cattrs.override(omit=True), - allow_recommending_already_measured=cattrs.override(omit=True), - ), - ), -) +converter.register_unstructure_hook(MetaRecommender, unstructure_base) converter.register_structure_hook( MetaRecommender, get_base_structure_hook(MetaRecommender) ) diff --git a/baybe/recommenders/naive.py b/baybe/recommenders/naive.py index 2ae51b1bb..b1b107977 100644 --- a/baybe/recommenders/naive.py +++ b/baybe/recommenders/naive.py @@ -1,11 +1,10 @@ """Naive recommender for hybrid spaces.""" import gc -import warnings from typing import ClassVar import pandas as pd -from attrs import define, evolve, field, fields +from attrs import define, field from typing_extensions import override from baybe.objectives.base import Objective @@ -48,36 +47,6 @@ class NaiveHybridSpaceRecommender(PureRecommender): """The recommender used for the continuous subspace. Default: :class:`baybe.recommenders.pure.bayesian.botorch.BotorchRecommender`""" - def __attrs_post_init__(self): - """Validate if flags are synchronized and overrides them otherwise.""" - if ( - flag := self.allow_recommending_already_measured - ) != self.disc_recommender.allow_recommending_already_measured: - warnings.warn( - f"The value of " - f"'{fields(self.__class__).allow_recommending_already_measured.name}' " - f"differs from what is specified in the discrete recommender. " - f"The value of the discrete recommender will be ignored." - ) - self.disc_recommender = evolve( - self.disc_recommender, - allow_recommending_already_measured=flag, - ) - - if ( - flag := self.allow_repeated_recommendations - ) != self.disc_recommender.allow_repeated_recommendations: - warnings.warn( - f"The value of " - f"'{fields(self.__class__).allow_repeated_recommendations.name}' " - f"differs from what is specified in the discrete recommender. " - f"The value of the discrete recommender will be ignored." - ) - self.disc_recommender = evolve( - self.disc_recommender, - allow_repeated_recommendations=flag, - ) - @override def recommend( self, @@ -122,12 +91,8 @@ def recommend( cont_part = searchspace.continuous.sample_uniform(1) cont_part_tensor = to_tensor(cont_part).unsqueeze(-2) - # Get discrete candidates. The metadata flags are ignored since the search space - # is hybrid - candidates_exp, _ = searchspace.discrete.get_candidates( - allow_repeated_recommendations=True, - allow_recommending_already_measured=True, - ) + # Get discrete candidates + candidates_exp, _ = searchspace.discrete.get_candidates() # We now check whether the discrete recommender is bayesian. if isinstance(self.disc_recommender, BayesianRecommender): diff --git a/baybe/recommenders/pure/base.py b/baybe/recommenders/pure/base.py index e84c9c753..90532e211 100644 --- a/baybe/recommenders/pure/base.py +++ b/baybe/recommenders/pure/base.py @@ -2,13 +2,13 @@ import gc from abc import ABC -from typing import ClassVar +from typing import ClassVar, NoReturn import pandas as pd from attrs import define, field from typing_extensions import override -from baybe.exceptions import NotEnoughPointsLeftError +from baybe.exceptions import DeprecationError, NotEnoughPointsLeftError from baybe.objectives.base import Objective from baybe.recommenders.base import RecommenderProtocol from baybe.searchspace import SearchSpace @@ -16,6 +16,11 @@ from baybe.searchspace.core import SearchSpaceType from baybe.searchspace.discrete import SubspaceDiscrete +_DEPRECATION_ERROR_MESSAGE = ( + "The attribute '{}' is no longer available for recommenders. " + "All 'allow_*' flags are now handled by `baybe.campaign.Campaign`." +) + # TODO: Slots are currently disabled since they also block the monkeypatching necessary # to use `register_hooks`. Probably, we need to update our documentation and @@ -24,23 +29,63 @@ class PureRecommender(ABC, RecommenderProtocol): """Abstract base class for all pure recommenders.""" - # Class variables compatibility: ClassVar[SearchSpaceType] """Class variable reflecting the search space compatibility.""" - # Object variables - allow_repeated_recommendations: bool = field(default=False, kw_only=True) - """Allow to make recommendations that were already recommended earlier. - This only has an influence in discrete search spaces.""" + _deprecated_allow_repeated_recommendations: bool = field( + alias="allow_repeated_recommendations", + default=None, + kw_only=True, + ) + "Deprecated! Now handled by :class:`baybe.campaign.Campaign`." + + _deprecated_allow_recommending_already_measured: bool = field( + alias="allow_recommending_already_measured", + default=None, + kw_only=True, + ) + "Deprecated! Now handled by :class:`baybe.campaign.Campaign`." + + _deprecated_allow_recommending_pending_experiments: bool = field( + alias="allow_recommending_pending_experiments", + default=None, + kw_only=True, + ) + "Deprecated! Now handled by :class:`baybe.campaign.Campaign`." + + def __attrs_post_init__(self): + if ( + self._deprecated_allow_repeated_recommendations is not None + or self._deprecated_allow_recommending_already_measured is not None + or self._deprecated_allow_recommending_pending_experiments is not None + ): + raise DeprecationError( + "Passing 'allow_*' flags to recommenders is no longer supported. " + "These are now handled by `baybe.campaign.Campaign`. " + "(Note: 'allow_repeated_recommendations' has been renamed to " + "'allow_recommending_already_recommended'.)" + ) + + @property + def allow_repeated_recommendations(self) -> NoReturn: + """Deprecated!""" + raise DeprecationError( + _DEPRECATION_ERROR_MESSAGE.format("allow_repeated_recommendations") + ) - allow_recommending_already_measured: bool = field(default=True, kw_only=True) - """Allow to make recommendations that were measured previously. - This only has an influence in discrete search spaces.""" + @property + def allow_recommending_already_measured(self) -> NoReturn: + """Deprecated!""" + raise DeprecationError( + _DEPRECATION_ERROR_MESSAGE.format("allow_recommending_already_measured") + ) - allow_recommending_pending_experiments: bool = field(default=False, kw_only=True) - """Allow `pending_experiments` to be part of the recommendations. If set to `False`, - the corresponding points will be removed from the candidates. This only has an - influence in discrete search spaces.""" + @property + def allow_recommending_pending_experiments(self) -> NoReturn: + """Deprecated!""" + raise DeprecationError( + _DEPRECATION_ERROR_MESSAGE.format("allow_recommending_pending_experiments") + ) @override def recommend( @@ -191,19 +236,7 @@ def _recommend_with_discrete_parts( is_hybrid_space = searchspace.type is SearchSpaceType.HYBRID # Get discrete candidates - # Repeated recommendations are always allowed for hybrid spaces - # Pending experiments are excluded for discrete spaces unless configured - # differently. - dont_exclude_pending = ( - is_hybrid_space or self.allow_recommending_pending_experiments - ) - candidates_exp, _ = searchspace.discrete.get_candidates( - allow_repeated_recommendations=is_hybrid_space - or self.allow_repeated_recommendations, - allow_recommending_already_measured=is_hybrid_space - or self.allow_recommending_already_measured, - exclude=None if dont_exclude_pending else pending_experiments, - ) + candidates_exp, _ = searchspace.discrete.get_candidates() # TODO: Introduce new flag to recommend batches larger than the search space @@ -212,10 +245,7 @@ def _recommend_with_discrete_parts( if (not is_hybrid_space) and (len(candidates_exp) < batch_size): raise NotEnoughPointsLeftError( f"Using the current settings, there are fewer than {batch_size} " - "possible data points left to recommend. This can happen " - "when all candidates have already been measured/recommended while " - "`allow_repeated_recommendations'/'allow_recommending_already_measured' " # noqa: E501 - "are set to `False`." + f"possible data points left to recommend." ) # Get recommendations diff --git a/baybe/recommenders/pure/nonpredictive/base.py b/baybe/recommenders/pure/nonpredictive/base.py index 12c3f1074..6f1a249b5 100644 --- a/baybe/recommenders/pure/nonpredictive/base.py +++ b/baybe/recommenders/pure/nonpredictive/base.py @@ -5,13 +5,13 @@ from abc import ABC import pandas as pd -from attrs import define, fields +from attrs import define from typing_extensions import override from baybe.exceptions import UnusedObjectWarning from baybe.objectives.base import Objective from baybe.recommenders.pure.base import PureRecommender -from baybe.searchspace.core import SearchSpace, SearchSpaceType +from baybe.searchspace.core import SearchSpace @define @@ -41,16 +41,10 @@ def recommend( f"consider any objectives, meaning that the argument is ignored.", UnusedObjectWarning, ) - if (pending_experiments is not None) and ( - self.allow_recommending_pending_experiments - or searchspace.type is not SearchSpaceType.DISCRETE - ): + if pending_experiments is not None: warnings.warn( f"Pending experiments were provided but the selected recommender " - f"'{self.__class__.__name__}' only utilizes this information for " - f"purely discrete spaces and " - f"{fields(self.__class__).allow_recommending_pending_experiments.name}" - f"=False.", + f"'{self.__class__.__name__}' does not use this information.", UnusedObjectWarning, ) return super().recommend( @@ -58,7 +52,7 @@ def recommend( searchspace=searchspace, objective=objective, measurements=measurements, - pending_experiments=pending_experiments, + pending_experiments=None, ) diff --git a/baybe/recommenders/pure/nonpredictive/sampling.py b/baybe/recommenders/pure/nonpredictive/sampling.py index 6cdeadef2..869781adf 100644 --- a/baybe/recommenders/pure/nonpredictive/sampling.py +++ b/baybe/recommenders/pure/nonpredictive/sampling.py @@ -34,7 +34,7 @@ def _recommend_hybrid( if searchspace.type == SearchSpaceType.CONTINUOUS: return cont_random - disc_candidates, _ = searchspace.discrete.get_candidates(True, True) + disc_candidates, _ = searchspace.discrete.get_candidates() # TODO decide mechanism if number of possible discrete candidates is smaller # than batch size diff --git a/baybe/searchspace/_annotated.py b/baybe/searchspace/_annotated.py deleted file mode 100644 index c798b845c..000000000 --- a/baybe/searchspace/_annotated.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Search spaces with metadata.""" - -import pandas as pd -from attrs import asdict, define, field -from typing_extensions import Self, override - -from baybe.searchspace import SubspaceDiscrete -from baybe.utils.boolean import eq_dataframe - - -@define -class AnnotatedSubspaceDiscrete(SubspaceDiscrete): - """An annotated search space carrying additional metadata.""" - - metadata: pd.DataFrame = field(kw_only=True, eq=eq_dataframe) - """The metadata.""" - - @classmethod - def from_subspace(cls, subspace: SubspaceDiscrete, metadata: pd.DataFrame) -> Self: - """Annotate an existing subspace with metadata.""" - return cls( - **asdict(subspace, filter=lambda attr, _: attr.init, recurse=False), - metadata=metadata, - ) - - @override - def get_candidates( - self, - allow_repeated_recommendations: bool = False, - allow_recommending_already_measured: bool = False, - exclude: pd.DataFrame | None = None, - ) -> tuple[pd.DataFrame, pd.DataFrame]: - from baybe.campaign import _EXCLUDED, _MEASURED, _RECOMMENDED - - # Exclude parts marked by metadata - mask_todrop = self.metadata[_EXCLUDED].copy() - if not allow_repeated_recommendations: - mask_todrop |= self.metadata[_RECOMMENDED] - if not allow_recommending_already_measured: - mask_todrop |= self.metadata[_MEASURED] - - # Remove additional excludes - if exclude is not None: - mask_todrop |= pd.merge(self.exp_rep, exclude, indicator=True, how="left")[ - "_merge" - ].eq("both") - - return self.exp_rep.loc[~mask_todrop], self.comp_rep.loc[~mask_todrop] diff --git a/baybe/searchspace/_filtered.py b/baybe/searchspace/_filtered.py new file mode 100644 index 000000000..b6949397e --- /dev/null +++ b/baybe/searchspace/_filtered.py @@ -0,0 +1,45 @@ +"""Search spaces with metadata.""" + +import numpy as np +import numpy.typing as npt +import pandas as pd +from attrs import asdict, cmp_using, define, field +from attrs.validators import instance_of +from typing_extensions import Self, override + +from baybe.searchspace import SubspaceDiscrete + + +@define +class FilteredSubspaceDiscrete(SubspaceDiscrete): + """A filtered search space representing a reduced candidate set.""" + + mask: npt.NDArray[np.bool_] = field( + validator=instance_of(np.ndarray), + kw_only=True, + eq=cmp_using(eq=np.array_equal), + ) + """The filtering mask. ``True`` denote elements to be kept.""" + + @mask.validator + def _validate_mask(self, _, value) -> None: + if not len(value) == len(self.exp_rep): + raise ValueError("Filter mask must match the size of the space.") + if not value.dtype == bool: + raise ValueError("Filter mask must only contain Boolean values.") + + @classmethod + def from_subspace( + cls, subspace: SubspaceDiscrete, mask: npt.NDArray[np.bool_] + ) -> Self: + """Filter an existing subspace.""" + return cls( + **asdict(subspace, filter=lambda attr, _: attr.init, recurse=False), + mask=mask, + ) + + @override + def get_candidates(self) -> tuple[pd.DataFrame, pd.DataFrame]: + mask_todrop = self._excluded.copy() + mask_todrop |= ~self.mask + return self.exp_rep.loc[~mask_todrop], self.comp_rep.loc[~mask_todrop] diff --git a/baybe/searchspace/discrete.py b/baybe/searchspace/discrete.py index bddd4461d..2792aa58d 100644 --- a/baybe/searchspace/discrete.py +++ b/baybe/searchspace/discrete.py @@ -598,40 +598,14 @@ def estimate_product_space_size( comp_rep_shape=(n_rows, n_cols_comp), ) - def get_candidates( - self, - allow_repeated_recommendations: bool = False, - allow_recommending_already_measured: bool = False, - exclude: pd.DataFrame | None = None, - ) -> tuple[pd.DataFrame, pd.DataFrame]: + def get_candidates(self) -> tuple[pd.DataFrame, pd.DataFrame]: """Return the set of candidate parameter settings that can be tested. - Args: - allow_repeated_recommendations: If ``True``, parameter settings that have - already been recommended in an earlier iteration are still considered - valid candidates. This is relevant, for instance, when an earlier - recommended parameter setting has not been measured by the user (for any - reason) after the corresponding recommendation was made. - allow_recommending_already_measured: If ``True``, parameters settings for - which there are already target values available are still considered as - valid candidates. - exclude: Points in experimental representation that should be excluded as - candidates. - Returns: The candidate parameter settings both in experimental and computational representation. """ - # Filter the search space down to the candidates - mask_todrop = self._excluded.copy() - - # Remove additional excludes - if exclude is not None: - mask_todrop |= pd.merge(self.exp_rep, exclude, indicator=True, how="left")[ - "_merge" - ].eq("both") - - return self.exp_rep.loc[~mask_todrop], self.comp_rep.loc[~mask_todrop] + return self.exp_rep.loc[~self._excluded], self.comp_rep.loc[~self._excluded] def transform( self, diff --git a/baybe/utils/basic.py b/baybe/utils/basic.py index 686ee7f9c..06c18d364 100644 --- a/baybe/utils/basic.py +++ b/baybe/utils/basic.py @@ -1,5 +1,6 @@ """Collection of small basic utilities.""" +import enum import functools import inspect from collections.abc import Callable, Collection, Iterable, Sequence @@ -16,6 +17,23 @@ _U = TypeVar("_U") +class UnspecifiedType(enum.Enum): + """Sentinel indicating an unspecified value when `None` is ambiguous.""" + + UNSPECIFIED = "UNSPECIFIED" + + @override + def __repr__(self): + return "UNSPECIFIED" + + def __bool__(self): + raise NotImplementedError(f"'{self!r}' has no Boolean representation.") + + +UNSPECIFIED = UnspecifiedType.UNSPECIFIED +"""Sentinel indicating an unspecified value when `None` is ambiguous.""" + + @dataclass(frozen=True, repr=False) class Dummy: """Placeholder element for array-like data types. diff --git a/docs/conf.py b/docs/conf.py index 403322ffe..7963cd946 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -125,6 +125,8 @@ (r"ref:obj", "baybe.surrogates.base.ModelContext"), # Ignore custom class properties (r"py:obj", "baybe.acquisition.acqfs.*.is_mc"), + # Other + (r"py:obj", "baybe.utils.basic.UnspecifiedType.UNSPECIFIED"), ] # Ignore the following links when checking inks for viability diff --git a/docs/userguide/async.md b/docs/userguide/async.md index 337fd273b..11ac7cf76 100644 --- a/docs/userguide/async.md +++ b/docs/userguide/async.md @@ -45,19 +45,19 @@ function with `pending_experiments` will result in an ```{admonition} Supported Recommenders :class: important -For technical reasons, not every recommender is able to utilize `pending_experiments` in -the same way. For instance, + +For technical reasons, not every recommender is able to make use of `pending_experiments`. +For instance, [`BotorchRecommender`](baybe.recommenders.pure.bayesian.botorch.BotorchRecommender) -takes all pending experiments into account, even if they do not match exactly with any -point in the search space. +takes all pending experiments into account, even if they do not match exactly with +points in the search space. +By contrast, [Non-predictive recommenders](baybe.recommenders.pure.nonpredictive.base.NonPredictiveRecommender) like [`SKLearnClusteringRecommender`](baybe.recommenders.pure.nonpredictive.clustering.SKLearnClusteringRecommender)s, [`RandomRecommender`](baybe.recommenders.pure.nonpredictive.sampling.RandomRecommender) or [`FPSRecommender`](baybe.recommenders.pure.nonpredictive.sampling.FPSRecommender) -only take pending points into consideration if the recommender flag -[allow_recommending_pending_experiments](baybe.recommenders.pure.nonpredictive.base.NonPredictiveRecommender.allow_recommending_pending_experiments) -is set to `False`. In that case, the candidate space is stripped of pending experiments -that are exact matches with the search space, i.e. they will not even be considered. +do not consider `pending_experiments` at all and raise an +{class}`~baybe.exceptions.UnusedObjectWarning` when such points are passed. ``` Akin to `measurements` or `recommendations`, `pending_experiments` is a dataframe in diff --git a/docs/userguide/campaigns.md b/docs/userguide/campaigns.md index faa3dd5fd..737e0b4f2 100644 --- a/docs/userguide/campaigns.md +++ b/docs/userguide/campaigns.md @@ -113,6 +113,18 @@ experimentation is feasible in the first place, or whether the given time budget even allows for sequential runs. ``` +### Candidate Control in Discrete Spaces +For discrete search spaces, campaigns provide additional control over how the candidate +set of recommendable points is built based on the trajectory the campaign has taken so +far. This is done by setting the following Boolean flags: +- `allow_recommending_already_measured`: Controls whether points that have already been + measured points can be recommended. +- `allow_recommending_already_recommended`: Controls whether previously recommended points can + be recommended again. +- `allow_recommending_pending_experiments`: Controls whether points marked as + `pending_experiments` can be recommended (see [asynchronous + workflows](PENDING_EXPERIMENTS)). + ### Caching of recommendations The `Campaign` object caches the last batch of recommendations returned, in order to diff --git a/docs/userguide/recommenders.md b/docs/userguide/recommenders.md index 351c25d8e..b0259562d 100644 --- a/docs/userguide/recommenders.md +++ b/docs/userguide/recommenders.md @@ -14,20 +14,6 @@ While some pure recommenders are versatile and work across different types of se spaces, other are specifically designed for discrete or continuous spaces. The compatibility is indicated via the corresponding `compatibility` class variable. -```{admonition} Additional Options for Discrete Search Spaces -:class: note -For discrete search spaces, BayBE provides additional controls for pure recommenders: -- `allow_repeated_recommendations`: Controls whether a recommender is allowed to - recommend previous recommendations again. -- `allow_recommending_already_measured`: Controls whether a recommender is allowed to - recommend points that have already been measured. This only considers exact matches - to the search space. -- `allow_recommending_pending_experiments`: Controls whether a recommender is allowed - to recommend points that have been marked as `pending_experiments` - (see [asynchronous workflows](PENDING_EXPERIMENTS)). This only considers exact matches to the - search space. -``` - ### Bayesian Recommenders The Bayesian recommenders in BayBE are built on the foundation of the diff --git a/examples/Basics/recommenders.py b/examples/Basics/recommenders.py index 324654e1c..0a49a967f 100644 --- a/examples/Basics/recommenders.py +++ b/examples/Basics/recommenders.py @@ -88,16 +88,6 @@ ACQ_FUNCTION = "qEI" -### Other parameters - -# Two other boolean hyperparameters can be specified when creating a recommender object. -# The first one allows the recommendation of points that were already recommended previously. -# The second one allows the recommendation of points that have already been measured. -# Per default, they are set to `True`. - -ALLOW_REPEATED_RECOMMENDATIONS = True -ALLOW_RECOMMENDING_ALREADY_MEASURED = True - ### Creating the recommender object # To create the recommender object, each parameter described above can be specified as follows. @@ -107,10 +97,7 @@ recommender = TwoPhaseMetaRecommender( initial_recommender=INITIAL_RECOMMENDER, recommender=BotorchRecommender( - surrogate_model=SURROGATE_MODEL, - acquisition_function=ACQ_FUNCTION, - allow_repeated_recommendations=ALLOW_REPEATED_RECOMMENDATIONS, - allow_recommending_already_measured=ALLOW_RECOMMENDING_ALREADY_MEASURED, + surrogate_model=SURROGATE_MODEL, acquisition_function=ACQ_FUNCTION ), ) diff --git a/examples/Custom_Hooks/campaign_stopping.py b/examples/Custom_Hooks/campaign_stopping.py index e2c0cb239..915480ed0 100644 --- a/examples/Custom_Hooks/campaign_stopping.py +++ b/examples/Custom_Hooks/campaign_stopping.py @@ -143,10 +143,7 @@ def stop_on_PI( botorch_acqf = acqf.to_botorch( self._surrogate_model, searchspace, objective, measurements ) - _, candidates_comp_rep = searchspace.discrete.get_candidates( - allow_repeated_recommendations=self.allow_repeated_recommendations, - allow_recommending_already_measured=self.allow_recommending_already_measured, - ) + _, candidates_comp_rep = searchspace.discrete.get_candidates() comp_rep_tensor = to_tensor(candidates_comp_rep).unsqueeze(1) acqf_values = botorch_acqf(comp_rep_tensor) diff --git a/examples/Multi_Armed_Bandit/bernoulli_multi_armed_bandit.py b/examples/Multi_Armed_Bandit/bernoulli_multi_armed_bandit.py index fc1415f93..62a2dceaa 100644 --- a/examples/Multi_Armed_Bandit/bernoulli_multi_armed_bandit.py +++ b/examples/Multi_Armed_Bandit/bernoulli_multi_armed_bandit.py @@ -126,11 +126,7 @@ def simulate(acqf: AcquisitionFunction) -> SimulationResult: recommender = TwoPhaseMetaRecommender( initial_recommender=RandomRecommender(), recommender=BotorchRecommender( - surrogate_model=surrogate, - acquisition_function=acqf, - # The same arm can be pulled several times: - allow_repeated_recommendations=True, - allow_recommending_already_measured=True, + surrogate_model=surrogate, acquisition_function=acqf ), ) @@ -141,7 +137,14 @@ def simulate(acqf: AcquisitionFunction) -> SimulationResult: for mc in range(N_MC_RUNS): searchspace = parameter.to_searchspace() objective = target.to_objective() - campaign = Campaign(searchspace, objective, recommender) + campaign = Campaign( + searchspace, + objective, + recommender, + # The same arm can be pulled several times: + allow_recommending_already_recommended=True, + allow_recommending_already_measured=True, + ) for i in range(N_ITERATIONS): df = campaign.recommend(batch_size=1) diff --git a/examples/Serialization/create_from_config.py b/examples/Serialization/create_from_config.py index 9ca83a117..b60693494 100644 --- a/examples/Serialization/create_from_config.py +++ b/examples/Serialization/create_from_config.py @@ -77,12 +77,12 @@ "surrogate_model": { "type": "GaussianProcessSurrogate" }, - "acquisition_function": "qEI", - "allow_repeated_recommendations": false, - "allow_recommending_already_measured": false + "acquisition_function": "qEI" }, "switch_after": 1 - } + }, + "allow_recommending_already_recommended": false, + "allow_recommending_already_measured": false } """ ) diff --git a/examples/Serialization/validate_config.py b/examples/Serialization/validate_config.py index c6aa37252..84465fdea 100644 --- a/examples/Serialization/validate_config.py +++ b/examples/Serialization/validate_config.py @@ -76,12 +76,12 @@ "surrogate_model": { "type": "GaussianProcessSurrogate" }, - "acquisition_function": "qEI", - "allow_repeated_recommendations": false, - "allow_recommending_already_measured": false + "acquisition_function": "qEI" }, "switch_after": 1 - } + }, + "allow_recommending_already_recommended": false, + "allow_recommending_already_measured": false } """ ) @@ -147,7 +147,7 @@ "type": "GaussianProcessSurrogate" }, "acquisition_function": "qEI", - "allow_repeated_recommendations": false, + "allow_recommending_already_recommended": false, "allow_recommending_already_measured": false } } diff --git a/tests/conftest.py b/tests/conftest.py index c82630cd4..5881e0ee0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -648,53 +648,19 @@ def fixture_default_surrogate_model(request, kernel): return GaussianProcessSurrogate(kernel_or_factory=kernel) -@pytest.fixture(name="allow_repeated_recommendations") -def fixture_allow_repeated_recommendations(): - return False - - -@pytest.fixture(name="allow_recommending_already_measured") -def allow_recommending_already_measured(): - return True - - -@pytest.fixture(name="allow_recommending_pending_experiments") -def fixture_allow_recommending_pending_experiments(): - return False - - @pytest.fixture(name="initial_recommender") -def fixture_initial_recommender( - allow_recommending_already_measured, - allow_repeated_recommendations, - allow_recommending_pending_experiments, -): +def fixture_initial_recommender(): """The default initial recommender to be used if not specified differently.""" - return RandomRecommender( - allow_repeated_recommendations=allow_repeated_recommendations, - allow_recommending_already_measured=allow_recommending_already_measured, - allow_recommending_pending_experiments=allow_recommending_pending_experiments, - ) + return RandomRecommender() @pytest.fixture(name="recommender") -def fixture_recommender( - initial_recommender, - surrogate_model, - acqf, - allow_repeated_recommendations, - allow_recommending_already_measured, - allow_recommending_pending_experiments, -): +def fixture_recommender(initial_recommender, surrogate_model, acqf): """The default recommender to be used if not specified differently.""" return TwoPhaseMetaRecommender( initial_recommender=initial_recommender, recommender=BotorchRecommender( - surrogate_model=surrogate_model, - acquisition_function=acqf, - allow_repeated_recommendations=allow_repeated_recommendations, - allow_recommending_already_measured=allow_recommending_already_measured, - allow_recommending_pending_experiments=allow_recommending_pending_experiments, + surrogate_model=surrogate_model, acquisition_function=acqf ), ) @@ -772,9 +738,7 @@ def fixture_default_config(): }, "recommender": { "type": "BotorchRecommender", - "acquisition_function": "qEI", - "allow_repeated_recommendations": false, - "allow_recommending_already_measured": false + "acquisition_function": "qEI" }, "switch_after": 1 } diff --git a/tests/simulate_telemetry.py b/tests/simulate_telemetry.py index 14527c0b0..39f7793b4 100644 --- a/tests/simulate_telemetry.py +++ b/tests/simulate_telemetry.py @@ -72,10 +72,7 @@ mode="SINGLE", targets=[NumericalTarget(name="Yield", mode="MAX")] ), "recommender": TwoPhaseMetaRecommender( - recommender=BotorchRecommender( - allow_repeated_recommendations=False, - allow_recommending_already_measured=False, - ), + recommender=BotorchRecommender(), initial_recommender=RandomRecommender(), ), } diff --git a/tests/test_campaign.py b/tests/test_campaign.py index 2d80ed867..f501e04f5 100644 --- a/tests/test_campaign.py +++ b/tests/test_campaign.py @@ -1,5 +1,7 @@ """Tests features of the Campaign object.""" +from contextlib import nullcontext + import pandas as pd import pytest from pytest import param @@ -7,8 +9,13 @@ from baybe.campaign import _EXCLUDED, Campaign from baybe.constraints.conditions import SubSelectionCondition from baybe.constraints.discrete import DiscreteExcludeConstraint -from baybe.parameters.numerical import NumericalDiscreteParameter +from baybe.parameters.numerical import ( + NumericalContinuousParameter, + NumericalDiscreteParameter, +) +from baybe.searchspace.core import SearchSpaceType from baybe.searchspace.discrete import SubspaceDiscrete +from baybe.utils.basic import UNSPECIFIED from .conftest import run_iterations @@ -73,3 +80,36 @@ def test_candidate_toggling(constraints, exclude, complement): other = campaign._searchspace_metadata[_EXCLUDED].drop(index=idx) assert all(target == exclude) # must contain the updated values assert all(other != exclude) # must contain the original values + + +@pytest.mark.parametrize( + "flag", + [ + "allow_recommending_already_measured", + "allow_recommending_already_recommended", + "allow_recommending_pending_experiments", + ], + ids=lambda x: x.removeprefix("allow_recommending_"), +) +@pytest.mark.parametrize( + "space_type", + [SearchSpaceType.DISCRETE, SearchSpaceType.CONTINUOUS], + ids=lambda x: x.name, +) +@pytest.mark.parametrize( + "value", [True, False, param(UNSPECIFIED, id=repr(UNSPECIFIED))] +) +def test_setting_allow_flags(flag, space_type, value): + """Passed allow_* flags are rejected if incompatible with the search space type.""" + kwargs = {flag: value} + expect_error = (space_type is SearchSpaceType.DISCRETE) != ( + value is not UNSPECIFIED + ) + + if space_type is SearchSpaceType.DISCRETE: + parameter = NumericalDiscreteParameter("p", [0, 1]) + else: + parameter = NumericalContinuousParameter("p", [0, 1]) + + with pytest.raises(ValueError) if expect_error else nullcontext(): + Campaign(parameter, **kwargs) diff --git a/tests/test_deprecations.py b/tests/test_deprecations.py index 9ce45a704..9819e25ba 100644 --- a/tests/test_deprecations.py +++ b/tests/test_deprecations.py @@ -29,6 +29,7 @@ BotorchRecommender, SequentialGreedyRecommender, ) +from baybe.recommenders.pure.nonpredictive.sampling import RandomRecommender from baybe.searchspace.continuous import SubspaceContinuous from baybe.searchspace.discrete import SubspaceDiscrete from baybe.searchspace.validation import get_transform_parameters @@ -284,3 +285,21 @@ def test_migrated_metadata_attribute(): NumericalDiscreteParameter("p", [0, 1]) ) subspace.metadata + + +@pytest.mark.parametrize( + "flag", + [ + "allow_repeated_recommendations", + "allow_recommending_already_measured", + "allow_recommending_pending_experiments", + ], +) +@pytest.mark.parametrize("recommender_cls", [RandomRecommender, BotorchRecommender]) +def test_migrated_allow_flags(flag, recommender_cls): + """Passing and accessing the migrated 'allow_*' flags raises an error.""" + with pytest.raises(DeprecationError, match=r"Passing 'allow_\*' flags"): + recommender_cls(**{flag: True}) + + with pytest.raises(DeprecationError, match=f"The attribute '{flag}' is no longer"): + getattr(recommender_cls(), flag) diff --git a/tests/test_integration.py b/tests/test_integration.py index a0d1f2e96..7daa3fce1 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -12,8 +12,7 @@ from baybe.utils.basic import get_subclasses nonpredictive_recommenders = [ - param(cls(allow_recommending_already_measured=True), id=cls.__name__) - for cls in get_subclasses(NonPredictiveRecommender) + param(cls(), id=cls.__name__) for cls in get_subclasses(NonPredictiveRecommender) ] p1 = NumericalDiscreteParameter("p1", [1, 2]) diff --git a/tests/test_iterations.py b/tests/test_iterations.py index fe056469f..93da8c4c4 100644 --- a/tests/test_iterations.py +++ b/tests/test_iterations.py @@ -316,7 +316,5 @@ def test_meta_recommenders(campaign, n_iterations, batch_size): ) @pytest.mark.parametrize("batch_size", [1]) @pytest.mark.parametrize("target_names", [["Target_binary"]]) -@pytest.mark.parametrize("allow_repeated_recommendations", [True]) -@pytest.mark.parametrize("allow_recommending_already_measured", [True]) def test_multi_armed_bandit(campaign, n_iterations, batch_size): run_iterations(campaign, n_iterations, batch_size, add_noise=False) diff --git a/tests/test_pending_experiments.py b/tests/test_pending_experiments.py index 961dd8a39..3841f9456 100644 --- a/tests/test_pending_experiments.py +++ b/tests/test_pending_experiments.py @@ -17,6 +17,7 @@ PAMClusteringRecommender, TwoPhaseMetaRecommender, ) +from baybe.searchspace.core import SearchSpaceType from baybe.utils.basic import get_subclasses from baybe.utils.dataframe import add_fake_measurements, add_parameter_noise from baybe.utils.random import temporary_seed @@ -25,86 +26,67 @@ _continuous_params = ["Conti_finite1", "Conti_finite2", "Conti_finite3"] _hybrid_params = ["Categorical_1", "Num_disc_1", "Conti_finite1", "Conti_finite2"] -# Repeated recommendations explicitly need to be allowed or the potential overlap will -# be avoided trivially -_flags = dict( - allow_repeated_recommendations=True, - allow_recommending_already_measured=True, -) - @pytest.mark.parametrize( "parameter_names, recommender", [ param( _discrete_params, - FPSRecommender(**_flags), + FPSRecommender(), id="fps_discrete", ), - param(_discrete_params, PAMClusteringRecommender(**_flags), id="pam_discrete"), + param(_discrete_params, PAMClusteringRecommender(), id="pam_discrete"), param( _discrete_params, - KMeansClusteringRecommender(**_flags), + KMeansClusteringRecommender(), id="kmeans_discrete", ), param( _discrete_params, - GaussianMixtureClusteringRecommender(**_flags), + GaussianMixtureClusteringRecommender(), id="gm_discrete", ), param( _discrete_params, - TwoPhaseMetaRecommender(recommender=BotorchRecommender(**_flags)), + TwoPhaseMetaRecommender(recommender=BotorchRecommender()), id="botorch_discrete", ), param( _continuous_params, - TwoPhaseMetaRecommender(recommender=BotorchRecommender(**_flags)), + TwoPhaseMetaRecommender(recommender=BotorchRecommender()), id="botorch_continuous", ), param( _hybrid_params, - TwoPhaseMetaRecommender(recommender=BotorchRecommender(**_flags)), + TwoPhaseMetaRecommender(recommender=BotorchRecommender()), id="botorch_hybrid", ), param( _discrete_params, - TwoPhaseMetaRecommender( - recommender=BotorchRecommender( - **_flags, allow_recommending_pending_experiments=True - ) - ), + TwoPhaseMetaRecommender(recommender=BotorchRecommender()), id="botorch_discrete_allow", ), param( _continuous_params, - TwoPhaseMetaRecommender( - recommender=BotorchRecommender( - **_flags, allow_recommending_pending_experiments=True - ) - ), + TwoPhaseMetaRecommender(recommender=BotorchRecommender()), id="botorch_continuous_allow", ), param( _hybrid_params, - TwoPhaseMetaRecommender( - recommender=BotorchRecommender( - **_flags, allow_recommending_pending_experiments=True - ) - ), + TwoPhaseMetaRecommender(recommender=BotorchRecommender()), id="botorch_hybrid_allow", ), param( _discrete_params, NaiveHybridSpaceRecommender( - disc_recommender=FPSRecommender(**_flags), **_flags + disc_recommender=FPSRecommender(), ), id="naive1_discrete", ), param( _discrete_params, NaiveHybridSpaceRecommender( - disc_recommender=KMeansClusteringRecommender(**_flags), **_flags + disc_recommender=KMeansClusteringRecommender(), ), id="naive2_discrete", ), @@ -115,6 +97,12 @@ def test_pending_points(campaign, batch_size): """Test there is no recommendation overlap if pending experiments are specified.""" warnings.filterwarnings("ignore", category=UnusedObjectWarning) + # Repeated recommendations explicitly need to be allowed or the potential overlap + # will be avoided trivially + if campaign.searchspace.type == SearchSpaceType.DISCRETE: + campaign.allow_recommending_already_recommended = True + campaign.allow_recommending_already_measured = True + # Perform a fake first iteration rec = campaign.recommend(batch_size) add_fake_measurements(rec, campaign.targets)