diff --git a/docs/dev/add_metric.md b/docs/dev/add_metric.md index ee70bb239..b118b7917 100644 --- a/docs/dev/add_metric.md +++ b/docs/dev/add_metric.md @@ -27,6 +27,7 @@ Definitions to keep in mind when comparing with other tools: - **IGD:** `igd` is the mean distance from each reference point to the front (p = 1, Coello and Cortés, as in pymoo). jMetal's default IGD is `sqrt(sum d^2) / n`, which gives smaller values. - **Multiplicative epsilon:** `epsilon_mult` needs strictly positive values in the front and in the reference front. Shift both explicitly when objectives can be 0, as on ZDT and DTLZ fronts. - **Exact fallback cost:** without MooCore, the exact hypervolume for four or more objectives grows exponentially with the number of objectives. SMS-EMOA and IBEA with the hypervolume indicator need the `[compute]` extra in practice from M = 4. +- **Approximate hypervolume:** **Development after 1.4.0:** `HVIndicator` and `HVApproxIndicator` (`hv_approx`) are exported by the experimental `vamos.ux.api` and documented in [Analysis and Visualization](../topics/analysis.md). `hv_approx` passes `samples`, `seed` and `method` (one of `HV_APPROX_METHODS`; `None` is MooCore's default) to `moocore.hv_approx` and records them in `details`; an unknown method raises `ValueError`. Do not persist a second metric file beside a canonical run. Bounded scalar metrics that belong to a run go in the manifest outcome; study tables are derived exports. diff --git a/docs/topics/analysis.md b/docs/topics/analysis.md index 77957760f..8d1188857 100644 --- a/docs/topics/analysis.md +++ b/docs/topics/analysis.md @@ -18,6 +18,56 @@ python -m pip install "vamos-optimization[analysis]==1.4.0" For environment setup and other extras, see [Installation](../guide/installation.md). +## Hypervolume, exact and approximate + +**Development after 1.4.0:** `vamos.ux.api` exports two hypervolume +indicators, `HVIndicator` (the exact value) and `HVApproxIndicator` (an +approximation), both computed by MooCore, which the `compute` extra installs. +Both take an explicit reference point, minimize every objective unless +`compute(..., maximise=True)` says otherwise, ignore dominated points and points +outside the reference box, and return an `IndicatorResult` with `value` and +`details`. Larger values are better. + +The cost of the exact hypervolume grows exponentially with the number of +objectives. From about six objectives it can take seconds to minutes: on one +laptop, fronts near DTLZ2's sphere of 2000 points with six objectives, 400 with +seven and 200 with eight took 17, 16 and 32 seconds, and the approximation of +each, with 10 000 samples, a few hundredths of a second. Use `HVApproxIndicator` +there; `compute(front, samples=..., seed=..., method=...)` sets it up: + +- `samples` is the number of directions of the estimate, 10 000 by default. + More samples are slower and usually more accurate. +- `method` is one of `HV_APPROX_METHODS`. `"Rphi-FWE+"` (MooCore's default + since MooCore 0.3.0) and `"DZ2019-HW"` are deterministic; `"DZ2019-MC"` + samples its directions at random. `None`, the default, uses MooCore's + default. +- `seed`, an integer or a NumPy `Generator`, seeds `"DZ2019-MC"`; the + deterministic methods ignore it. With `seed=None` that method gives another + value on each call. +- `details` records `reference_point`, `samples`, `seed` and the `method` used. + +On 286 points of DTLZ2's front with four objectives and the reference point +1.1, the deterministic methods with 10 000 samples are within 0.04 % of the +exact value, and `"DZ2019-MC"` with 100 000 samples within 0.3 %. Report the +method and the samples with each approximate value, and do not compare an +approximate value with an exact one. + +```python +import numpy as np + +from vamos.ux.api import HVApproxIndicator, HVIndicator + +# 200 points of DTLZ2's front with six objectives. +directions = np.abs(np.random.default_rng(0).normal(size=(200, 6))) +front = directions / np.linalg.norm(directions, axis=1, keepdims=True) +reference_point = np.full(6, 1.1) + +exact = HVIndicator(reference_point=reference_point).compute(front) +approx = HVApproxIndicator(reference_point=reference_point).compute(front, samples=100_000, method="DZ2019-MC", seed=1) +print(f"exact {exact.value:.4f}, approximate {approx.value:.4f}") +print(approx.details["method"], approx.details["seed"], approx.details["samples"]) +``` + ## Statistical comparison For real saved observations, follow [T07](../tutorials/t07-quality-and-budget.md) diff --git a/src/vamos/foundation/quality_indicators/__init__.py b/src/vamos/foundation/quality_indicators/__init__.py index 2e05b145e..71a5580bf 100644 --- a/src/vamos/foundation/quality_indicators/__init__.py +++ b/src/vamos/foundation/quality_indicators/__init__.py @@ -5,6 +5,7 @@ ) from .hypervolume import compute_hypervolume from .moocore_indicators import ( + HV_APPROX_METHODS, AvgHausdorffIndicator, EpsilonAdditiveIndicator, EpsilonMultiplicativeIndicator, @@ -37,5 +38,6 @@ "EpsilonMultiplicativeIndicator", "AvgHausdorffIndicator", "HVApproxIndicator", + "HV_APPROX_METHODS", "WHVRectIndicator", ] diff --git a/src/vamos/foundation/quality_indicators/moocore_indicators.py b/src/vamos/foundation/quality_indicators/moocore_indicators.py index 3909986e2..d28b3cec9 100644 --- a/src/vamos/foundation/quality_indicators/moocore_indicators.py +++ b/src/vamos/foundation/quality_indicators/moocore_indicators.py @@ -17,6 +17,7 @@ from __future__ import annotations +import inspect from collections.abc import Sequence from dataclasses import dataclass, field from typing import Any, Protocol @@ -217,8 +218,27 @@ def compute( return result +# MooCore's methods of hv_approx: "Rphi-FWE+" (since MooCore 0.3.0) and "DZ2019-HW" are deterministic, and +# "DZ2019-MC" samples at random from its seed. +HV_APPROX_METHODS = ("Rphi-FWE+", "DZ2019-HW", "DZ2019-MC") + + @dataclass class HVApproxIndicator(QualityIndicator): + """Approximate hypervolume: MooCore's ``hv_approx``. + + The cost of the exact hypervolume (``hv``) grows exponentially with the number of objectives: from + about six objectives, a front of a few hundred to a few thousand points can take from seconds to + minutes. ``hv_approx`` estimates it as an integral over ``samples`` directions (10 000 by default; + Deng and Zhang, 2019), with the same reference point and the same handling of dominated points and of + points outside the reference box. + + Development after 1.4.0: ``compute`` also takes ``seed`` (an integer, a NumPy ``Generator`` or + ``None``), which only ``method="DZ2019-MC"`` uses, and ``method`` (one of ``HV_APPROX_METHODS``; + ``None``, the default, is MooCore's default: ``"Rphi-FWE+"`` since MooCore 0.3.0, ``"DZ2019-HW"`` + before). The result's details record ``samples``, ``seed`` and the ``method`` used. + """ + reference_point: ArrayLike | None = None name: str = "hv_approx" @@ -233,8 +253,14 @@ def compute( F = _front(front) ref = _reference_point(self.reference_point, F.shape[1], name=self.name) samples = int(kwargs.get("samples", 10000)) - val = float(moocore.hv_approx(F, ref=ref, maximise=_normalize_maximise(maximise), nsamples=samples)) - return IndicatorResult(value=val, details={"reference_point": ref, "samples": samples}) + seed = kwargs.get("seed") + method = kwargs.get("method") + if method is None: + method = str(inspect.signature(moocore.hv_approx).parameters["method"].default) + elif method not in HV_APPROX_METHODS: + raise ValueError(f"hv_approx method must be one of {', '.join(map(repr, HV_APPROX_METHODS))}; got {method!r}.") + val = float(moocore.hv_approx(F, ref=ref, maximise=_normalize_maximise(maximise), nsamples=samples, seed=seed, method=method)) + return IndicatorResult(value=val, details={"reference_point": ref, "samples": samples, "seed": seed, "method": method}) @dataclass @@ -331,5 +357,6 @@ def get_indicator(name: str, **kwargs: Any) -> QualityIndicator: "EpsilonMultiplicativeIndicator", "AvgHausdorffIndicator", "HVApproxIndicator", + "HV_APPROX_METHODS", "WHVRectIndicator", ] diff --git a/src/vamos/ux/api.py b/src/vamos/ux/api.py index e05e0b5c8..ef4f6117e 100644 --- a/src/vamos/ux/api.py +++ b/src/vamos/ux/api.py @@ -4,6 +4,12 @@ from __future__ import annotations +from vamos.foundation.quality_indicators.moocore_indicators import ( + HV_APPROX_METHODS, + HVApproxIndicator, + HVIndicator, + IndicatorResult, +) from vamos.ux.analysis.core_objective_reduction import ( ObjectiveReducer, ObjectiveReductionConfig, @@ -40,6 +46,11 @@ ) __all__ = [ + # Development after 1.4.0: quality indicators, the exact and the approximate hypervolume + "HVIndicator", + "HVApproxIndicator", + "HV_APPROX_METHODS", + "IndicatorResult", # Objective reduction "ObjectiveReductionConfig", "ObjectiveReducer", diff --git a/tests/architecture/public_api_snapshot.json b/tests/architecture/public_api_snapshot.json index 3af0091b3..9008a1249 100644 --- a/tests/architecture/public_api_snapshot.json +++ b/tests/architecture/public_api_snapshot.json @@ -181,6 +181,10 @@ ], "vamos.ux.api": [ "FriedmanResult", + "HVApproxIndicator", + "HVIndicator", + "HV_APPROX_METHODS", + "IndicatorResult", "MCDMResult", "ObjectiveReducer", "ObjectiveReductionConfig", diff --git a/tests/docs/smoke_manifest.py b/tests/docs/smoke_manifest.py index ba93c0b01..53d673e76 100644 --- a/tests/docs/smoke_manifest.py +++ b/tests/docs/smoke_manifest.py @@ -226,14 +226,14 @@ def build_my_algorithm(cfg: dict[str, Any], kernel: KernelBackend | None = None) import tempfile from pathlib import Path -if importlib.util.find_spec("matplotlib") is not None: +if importlib.util.find_spec("matplotlib") is not None and importlib.util.find_spec("moocore") is not None: text = Path("docs/topics/analysis.md").read_text(encoding="utf-8") blocks = [] for part in text.split("```python")[1:]: block, separator, _rest = part.partition("```") assert separator == "```" blocks.append(block.strip()) - assert len(blocks) == 4 + assert len(blocks) == 5 with tempfile.TemporaryDirectory() as tmp: tmp_path = Path(tmp) diff --git a/tests/docs/test_analysis_docs.py b/tests/docs/test_analysis_docs.py index d6a4c7cfc..a0005f609 100644 --- a/tests/docs/test_analysis_docs.py +++ b/tests/docs/test_analysis_docs.py @@ -15,6 +15,9 @@ def test_analysis_docs_match_public_contract() -> None: assert "`p_adjusted`, Holm's step-down adjustment" in text assert "MCDMResult" in text assert "knee_point_scores(F)" in text + # Development after 1.4.0: the exact and the approximate hypervolume. + assert "`HVApproxIndicator`" in text and "`HV_APPROX_METHODS`" in text + assert "`details` records `reference_point`, `samples`, `seed` and the `method` used" in text assert "topsis_scores" not in text assert "num_datasets=" not in text diff --git a/tests/foundation/test_moocore_indicators.py b/tests/foundation/test_moocore_indicators.py index 8dba7033e..7be6fad74 100644 --- a/tests/foundation/test_moocore_indicators.py +++ b/tests/foundation/test_moocore_indicators.py @@ -1,3 +1,5 @@ +import itertools + import numpy as np import pytest @@ -123,3 +125,70 @@ def test_outside_points_are_filtered_in_every_hv_path() -> None: if mi.has_moocore(): assert mi.HVIndicator(reference_point=ref).compute(points).value == pytest.approx(0.48) assert mi.HVContributionsIndicator(reference_point=ref).compute(points).value == pytest.approx([0.48, 0.0]) + + +def _dtlz2_front(n_obj: int, divisions: int) -> np.ndarray: + """DTLZ2's front: a Das-Dennis simplex lattice projected onto the unit sphere.""" + rows = [c for c in itertools.product(range(divisions + 1), repeat=n_obj - 1) if sum(c) <= divisions] + w = np.array([(*c, divisions - sum(c)) for c in rows], dtype=float) + return w / np.linalg.norm(w, axis=1, keepdims=True) + + +@pytest.mark.skipif(not mi.has_moocore(), reason="MooCore not installed") +@pytest.mark.parametrize(("method", "samples"), [(None, 10_000), ("Rphi-FWE+", 10_000), ("DZ2019-HW", 10_000), ("DZ2019-MC", 100_000)]) +def test_hv_approx_agrees_with_the_exact_hypervolume_on_dtlz2_with_four_objectives(method: str | None, samples: int) -> None: + # Development after 1.4.0: 286 points of DTLZ2's front with four objectives. + front = _dtlz2_front(4, 10) + ref = np.full(4, 1.1) + exact = mi.HVIndicator(reference_point=ref).compute(front).value + + approx = mi.HVApproxIndicator(reference_point=ref).compute(front, samples=samples, seed=7, method=method) + + assert approx.value == pytest.approx(exact, rel=0.01) + assert approx.details["samples"] == samples + + +@pytest.mark.skipif(not mi.has_moocore(), reason="MooCore not installed") +def test_hv_approx_passes_its_seed_to_moocore() -> None: + front = _dtlz2_front(4, 6) + indicator = mi.HVApproxIndicator(reference_point=np.full(4, 1.1)) + + first = indicator.compute(front, seed=1, method="DZ2019-MC") + again = indicator.compute(front, seed=1, method="DZ2019-MC") + other = indicator.compute(front, seed=2, method="DZ2019-MC") + + assert first.value == again.value + assert other.value != first.value + assert (first.details["seed"], first.details["method"]) == (1, "DZ2019-MC") + + +@pytest.mark.skipif(not mi.has_moocore(), reason="MooCore not installed") +def test_hv_approx_records_moocores_default_method() -> None: + import inspect + + import moocore + + front = _dtlz2_front(3, 4) + ref = np.full(3, 1.1) + + result = mi.HVApproxIndicator(reference_point=ref).compute(front) + + default = inspect.signature(moocore.hv_approx).parameters["method"].default + assert {key: result.details[key] for key in ("samples", "seed", "method")} == {"samples": 10_000, "seed": None, "method": default} + np.testing.assert_array_equal(result.details["reference_point"], ref) + assert result.value == moocore.hv_approx(front, ref=ref, nsamples=10_000, method=default) + + +@pytest.mark.skipif(not mi.has_moocore(), reason="MooCore not installed") +def test_hv_approx_rejects_an_unknown_method() -> None: + with pytest.raises(ValueError, match=r"'Rphi-FWE\+', 'DZ2019-HW', 'DZ2019-MC'"): + mi.HVApproxIndicator(reference_point=np.full(2, 1.1)).compute(_REFERENCE_FRONT, method="monte-carlo") + + +def test_the_analysis_facade_exports_the_exact_and_the_approximate_hypervolume() -> None: + from vamos.ux import api + + assert api.HVApproxIndicator is mi.HVApproxIndicator is type(mi.get_indicator("hv_approx")) + assert api.HVIndicator is mi.HVIndicator + assert api.IndicatorResult is mi.IndicatorResult + assert api.HV_APPROX_METHODS == mi.HV_APPROX_METHODS