Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cbranincurrin synthetic test function for cmoo #692

Open
wants to merge 14 commits into
base: develop
Choose a base branch
from
8 changes: 8 additions & 0 deletions docs/refs.bib
Original file line number Diff line number Diff line change
Expand Up @@ -504,3 +504,11 @@ @InProceedings{moss2023IPA
booktitle = {Proceedings of the Twenty-Fith International Conference on Artificial Intelligence and Statistics},
year = {2023},
}

@article{belakaria2019max,
title={Max-value entropy search for multi-objective bayesian optimization},
author={Belakaria, Syrine and Deshwal, Aryan and Doppa, Janardhan Rao},
journal={Advances in Neural Information Processing Systems},
volume={32},
year={2019}
}
86 changes: 73 additions & 13 deletions tests/unit/objectives/test_multi_objectives.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,21 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import Callable
from typing import Callable, Union

import numpy.testing as npt
import pytest
import tensorflow as tf

from tests.util.misc import TF_DEBUGGING_ERROR_TYPES
from trieste.objectives.multi_objectives import DTLZ1, DTLZ2, VLMOP2, MultiObjectiveTestProblem
from trieste.objectives.multi_objectives import (
DTLZ1,
DTLZ2,
VLMOP2,
ConstrainedBraninCurrin,
ConstrainedMultiObjectiveTestProblem,
MultiObjectiveTestProblem,
)
from trieste.types import TensorType


Expand Down Expand Up @@ -107,6 +114,47 @@ def test_dtlz2_has_expected_output(
npt.assert_allclose(f(test_x), expected, rtol=1e-4)


@pytest.mark.parametrize(
"test_x, expected_obj, expected_con, threshold",
[
(
tf.constant([[0.0, 0.0]]),
tf.constant([[308.12909601160663, 3.0]]),
tf.constant([[62.5]]),
0.0,
),
(
tf.constant([[0.5, 1.0]]),
tf.constant([[150.45202034083485, 4.609388478538837]]),
tf.constant([[-3.75]]),
10.0,
),
(
tf.constant([[[0.5, 1.0]], [[0.0, 0.0]]]),
tf.constant([[[150.45202034083485, 4.609388478538837]], [[308.12909601160663, 3.0]]]),
tf.constant([[[6.25]], [[62.5]]]),
0.0,
),
(
tf.constant([[0.5, 1.0], [0.0, 0.0]]),
tf.constant([[150.45202034083485, 4.609388478538837], [308.12909601160663, 3.0]]),
tf.constant([[11.25], [67.5]]),
-5.0,
),
],
)
def test_constrainedbranincurrin_has_expected_output(
test_x: TensorType,
expected_obj: TensorType,
expected_con: TensorType,
threshold: Union[TensorType, float],
) -> None:
f = ConstrainedBraninCurrin().objective
c = ConstrainedBraninCurrin().constraint
npt.assert_allclose(f(test_x), expected_obj, rtol=1e-5)
npt.assert_allclose(c(test_x, threshold), expected_con, rtol=1e-5)


@pytest.mark.parametrize(
"obj_type, input_dim, num_obj, gen_pf_num",
[
Expand Down Expand Up @@ -137,33 +185,40 @@ def test_gen_pareto_front_is_equal_to_math_defined(
(VLMOP2(2), tf.constant([[0.4, 0.2, 0.5]])),
(DTLZ1(3, 2), tf.constant([[0.3, 0.1]])),
(DTLZ2(5, 2), tf.constant([[0.3, 0.1]])),
(ConstrainedBraninCurrin(), tf.constant([[0.3, 0.2, 0.1]])),
],
)
def test_func_raises_specified_input_dim_not_align_with_actual_input_dim(
obj_inst: MultiObjectiveTestProblem, actual_x: TensorType
obj_inst: Union[MultiObjectiveTestProblem, ConstrainedMultiObjectiveTestProblem],
actual_x: TensorType,
) -> None:
with pytest.raises(TF_DEBUGGING_ERROR_TYPES):
obj_inst.objective(actual_x)
if isinstance(obj_inst, ConstrainedMultiObjectiveTestProblem):
with pytest.raises(TF_DEBUGGING_ERROR_TYPES):
obj_inst.constraint(actual_x)


@pytest.mark.parametrize(
TsingQAQ marked this conversation as resolved.
Show resolved Hide resolved
"problem, input_dim, num_obj",
"problem, input_dim, num_obj, num_con",
[
(VLMOP2(2), 2, 2),
(VLMOP2(10), 10, 2),
(DTLZ1(3, 2), 3, 2),
(DTLZ1(10, 5), 10, 5),
(DTLZ2(3, 2), 3, 2),
(DTLZ2(10, 5), 10, 5),
(VLMOP2(2), 2, 2, 0),
(VLMOP2(10), 10, 2, 0),
(DTLZ1(3, 2), 3, 2, 0),
(DTLZ1(10, 5), 10, 5, 0),
(DTLZ2(3, 2), 3, 2, 0),
(DTLZ2(10, 5), 10, 5, 0),
(ConstrainedBraninCurrin(), 2, 2, 1),
],
)
@pytest.mark.parametrize("num_obs", [1, 5, 10])
@pytest.mark.parametrize("dtype", [tf.float32, tf.float64])
def test_objective_has_correct_shape_and_dtype(
problem: MultiObjectiveTestProblem,
def test_objective_and_constraint_has_correct_shape_and_dtype(
problem: Union[MultiObjectiveTestProblem, ConstrainedMultiObjectiveTestProblem],
TsingQAQ marked this conversation as resolved.
Show resolved Hide resolved
input_dim: int,
num_obj: int,
num_obs: int,
num_con: int,
dtype: tf.DType,
) -> None:
x = problem.search_space.sample(num_obs)
Expand All @@ -177,5 +232,10 @@ def test_objective_has_correct_shape_and_dtype(
tf.debugging.assert_shapes([(x, [num_obs, input_dim])])
tf.debugging.assert_shapes([(y, [num_obs, num_obj])])

if isinstance(problem, ConstrainedMultiObjectiveTestProblem):
c = problem.constraint(x)
tf.debugging.assert_shapes([(c, [num_obs, num_con])])

assert pf.dtype == tf.float64 # default dtype
tf.debugging.assert_shapes([(pf, [num_obs * 2, num_obj])])
if tf.size(pf) != 0: # the problem has a valid `gen_pareto_optimal_points` method
tf.debugging.assert_shapes([(pf, [num_obs * 2, num_obj])])
97 changes: 95 additions & 2 deletions trieste/objectives/multi_objectives.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,14 @@
import math
from dataclasses import dataclass
from functools import partial
from typing import Optional

import tensorflow as tf
from typing_extensions import Protocol

from ..space import Box
from ..types import TensorType
from .single_objectives import ObjectiveTestProblem
from .single_objectives import ObjectiveTestProblem, branin


class GenParetoOptimalPoints(Protocol):
Expand All @@ -38,7 +39,21 @@ def __call__(self, n: int, seed: int | None = None) -> TensorType:
:param n: The number of pareto optimal points to be generated.
:param seed: An integer used to create a random seed for distributions that
used to generate pareto optimal points.
:return: The Pareto optimal points
:return: The Pareto optimal points.
"""


class Constraint(Protocol):
TsingQAQ marked this conversation as resolved.
Show resolved Hide resolved
"""A Protocol representing function returning constraint values given specified inputs."""

def __call__(self, x: TensorType, threshold: Optional[float] = 0.0) -> TensorType:
"""
return the constraint value given specified inputs `x` and `threshold`

:param x: The points at which to evaluate the function, with shape [..., d].
:param threshold: a feasibility threshold used to determine the constraint, by default 0 is
used as in the original problem.
:return: The constraint values.
"""


Expand All @@ -55,6 +70,17 @@ class MultiObjectiveTestProblem(ObjectiveTestProblem):
random number seed."""


@dataclass(frozen=True)
class ConstrainedMultiObjectiveTestProblem(MultiObjectiveTestProblem):
"""
Convenience container class for synthetic constrained multi-objective test functions, containing
additionally a constraint function.
"""

constraint: Constraint
"""The synthetic test function's constraints"""


def vlmop2(x: TensorType, d: int) -> TensorType:
"""
The VLMOP2 synthetic function.
Expand Down Expand Up @@ -236,3 +262,70 @@ def gen_pareto_optimal_points(n: int, seed: int | None = None) -> TensorType:
search_space=Box([0.0], [1.0]) ** d,
gen_pareto_optimal_points=gen_pareto_optimal_points,
)


def ConstrainedBraninCurrin() -> ConstrainedMultiObjectiveTestProblem:
"""
The ConstrainedBraninCurrin problem, typically evaluated over :math:`[0, 1]^2`.
See :cite:`belakaria2019max` and :cite:`daulton2020differentiable`
(the latter for adding the constraint) for details.

:return: The problem specification.
"""

def gen_pareto_optimal_points(n: int, seed: int | None = None) -> TensorType:
"""
return an empty tensor as there is no explicit way of defining
this problem's Pareto frontier.
"""
return tf.zeros(shape=0, dtype=tf.float64)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

so there is no known pareto front for this problem I take it?
can you please add a note to the docs of the problem above stating that and that this function will return an empty tensor please

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@uri-granta do you think this is ok or we should return NotImplementedError? for some problems pareto front will not be available

it's a bit burdensome, but what can be done is to compute offline approximate ground truth with say genetic algos with a large number of function evaluations, store a fixed number of points in the repo as a text file and then load that when the function is called

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Definitely clearer to raise a NotImplementedError with a message explaining that there is no known pareto front, rather than returning an empty "front".

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@TsingQAQ can you please raise an error here, but perhaps in the docs you can indicate to user a way to generate approximate ground truth

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure! for readability have created a new exception class NoAnalyticalParetoPointsError, please check if this is idea.


def evaluate_constraint(x: TensorType, threshold: Optional[float] = 0.0) -> TensorType:
TsingQAQ marked this conversation as resolved.
Show resolved Hide resolved
"""
The constraint of branincurrin problem, < ``threshold`` is feasible.

:param x: The points at which to evaluate the function, with shape [..., d].
:param threshold: a feasibility threshold used to determine the constraint, by default 0 is
used as in the original problem.
:raise ValueError (or InvalidArgumentError): If ``x`` has an invalid shape.
"""
x = x * (
tf.constant([10.0, 15.0], dtype=x.dtype) - tf.constant([-5.0, 0.0], dtype=x.dtype)
) + tf.constant([-5.0, 0.0], dtype=x.dtype)
return (x[..., :1] - 2.5) ** 2 + (x[..., 1:] - 7.5) ** 2 - 50 - threshold

return ConstrainedMultiObjectiveTestProblem(
name="ConstrainedBraninCurrin",
objective=branin_currin,
constraint=evaluate_constraint,
search_space=Box([0.0], [1.0]) ** 2,
gen_pareto_optimal_points=gen_pareto_optimal_points,
)


def branin_currin(x: TensorType) -> TensorType:
"""
The branincurrin synthetic function.

:param x: The points at which to evaluate the function, with shape [..., d].
:raise ValueError (or InvalidArgumentError): If ``x`` has an invalid shape.
"""
tf.debugging.assert_shapes([(x, (..., 2))])
return tf.concat([branin(x), currin(x)], axis=-1)
TsingQAQ marked this conversation as resolved.
Show resolved Hide resolved


def currin(x: TensorType) -> TensorType:
"""
The currin synthetic function

:param x: The points at which to evaluate the function, with shape [..., d].
:raise ValueError (or InvalidArgumentError): If ``x`` has an invalid shape.
"""
tf.debugging.assert_shapes([(x, (..., 2))])
return (
(1 - tf.math.exp(-0.5 * (1 / (x[..., 1] + 1e-100)))) # 1e-100 used for avoid zero division
* (
(2300 * x[..., 0] ** 3 + 1900 * x[..., 0] ** 2 + 2092 * x[..., 0] + 60)
/ (100 * x[..., 0] ** 3 + 500 * x[..., 0] ** 2 + 4 * x[..., 0] + 20)
)
)[..., tf.newaxis]