From 8f71c38e499a12d10a43eb5dbe9b2e54d9d41edb Mon Sep 17 00:00:00 2001 From: Drew Oldag Date: Wed, 8 Nov 2023 21:10:38 -0800 Subject: [PATCH] Wrapping the rest of the metrics in `metrics.py`. --- src/qp/metrics/base_metric_classes.py | 21 ++- src/qp/metrics/concrete_metric_classes.py | 218 +++++++++++++++++++++- 2 files changed, 229 insertions(+), 10 deletions(-) diff --git a/src/qp/metrics/base_metric_classes.py b/src/qp/metrics/base_metric_classes.py index c5faed3..2f12378 100644 --- a/src/qp/metrics/base_metric_classes.py +++ b/src/qp/metrics/base_metric_classes.py @@ -75,9 +75,9 @@ class BaseMetric(ABC): metric_input_type = MetricInputType.unknown # The type of input data expected for this metric metric_output_type = MetricOuputType.unknown # The form of the output data from this metric - def __init__(self, limit:tuple=(0.0, 3.0), dx:float=0.01) -> None: + def __init__(self, limits:tuple=(0.0, 3.0), dx:float=0.01) -> None: - self._limit = limit + self._limits = limits self._dx = dx @classmethod @@ -144,7 +144,24 @@ class PointToPointMetric(BaseMetric): metric_input_type = MetricInputType.point_to_point + def initialize(self): + raise NotImplementedError() + + def evaluate(self, estimate, reference): + raise NotImplementedError() + + def finalize(self): + raise NotImplementedError() class PointToDistMetric(BaseMetric): metric_input_type = MetricInputType.point_to_dist + + def initialize(self): + raise NotImplementedError() + + def evaluate(self, estimate, reference): + raise NotImplementedError() + + def finalize(self): + raise NotImplementedError() diff --git a/src/qp/metrics/concrete_metric_classes.py b/src/qp/metrics/concrete_metric_classes.py index fd83e63..3888d06 100644 --- a/src/qp/metrics/concrete_metric_classes.py +++ b/src/qp/metrics/concrete_metric_classes.py @@ -1,5 +1,20 @@ -from qp.metrics.base_metric_classes import MetricOuputType, DistToDistMetric, SingleEnsembleMetric -from qp.metrics.metrics import calculate_kld, calculate_moment +import numpy as np + +from qp.metrics.base_metric_classes import ( + MetricOuputType, + DistToDistMetric, + DistToPointMetric, + SingleEnsembleMetric, +) +from qp.metrics.metrics import ( + calculate_brier, + calculate_goodness_of_fit, + calculate_kld, + calculate_moment, + calculate_outlier_rate, + calculate_rmse, + calculate_rbpe, +) class MomentMetric(SingleEnsembleMetric): """Class wrapper around the `calculate_moment` function. @@ -8,10 +23,8 @@ class MomentMetric(SingleEnsembleMetric): metric_name = "moment" def __init__(self, moment_order:int=1, limits:tuple=(0.0, 3.0), dx:float=0.01) -> None: - super().__init__() + super().__init__(limits, dx) self._moment_order = moment_order - self._limits = limits - self._dx = dx def initialize(self) -> None: pass @@ -31,9 +44,7 @@ class KLDMetric(DistToDistMetric): metric_output_type = MetricOuputType.one_value_per_distribution def __init__(self, limits:tuple=(0.0, 3.0), dx:float=0.01) -> None: - super().__init__() - self._limits = limits - self._dx = dx + super().__init__(limits, dx) def initialize(self) -> None: pass @@ -43,3 +54,194 @@ def evaluate(self, estimate, reference) -> list: def finalize(self) -> None: pass + + +class RMSEMetric(DistToDistMetric): + """Class wrapper around the Root Mean Square Error metric + """ + + metric_name = "rmse" + metric_output_type = MetricOuputType.one_value_per_distribution + + def __init__(self, limits:tuple=(0.0, 3.0), dx:float=0.01) -> None: + super().__init__(limits, dx) + + def initialize(self) -> None: + pass + + def evaluate(self, estimate, reference) -> list: + return calculate_rmse(estimate, reference, self._limits, self._dx) + + def finalize(self) -> None: + pass + + +class RBPEMetric(SingleEnsembleMetric): + """Class wrapper around the Risk Based Point Estimate metric. + """ + + metric_name = 'rbpe' + metric_output_type = MetricOuputType.one_value_per_distribution + + def __init__(self, limits:tuple=(np.inf, np.inf)) -> None: + super().__init__(limits) + + def initialize(self) -> None: + pass + + def evaluate(self, estimate) -> list: + return calculate_rbpe(estimate, self._limits) + + def finalize(self) -> None: + pass + + +#! Should this be implemented as `DistToPointMetric` or `DistToDistMetric` ??? +class BrierMetric(DistToPointMetric): + """Class wrapper around the calculate_brier function. (Which itself is a + wrapper around the `Brier` metric evaluator class). + """ + + metric_name = 'brier' + metric_output_type = MetricOuputType.one_value_per_distribution + + def __init__(self, limits:tuple=(0.0, 3.0), dx:float=0.01) -> None: + super().__init__(limits, dx) + + def initialize(self) -> None: + pass + + def evaluate(self, estimate, reference) -> list: + return calculate_brier(estimate, reference, self._limits, self._dx) + + def finalize(self) -> None: + pass + + +class OutlierMetric(SingleEnsembleMetric): + """Class wrapper around the outlier calculation metric. + """ + + metric_name = 'outlier' + metric_output_type = MetricOuputType.one_value_per_distribution + + def __init__(self, cdf_limits:tuple = (0.0001, 0.9999)) -> None: + super().__init__() + self._cdf_limits=cdf_limits + + def initialize(self) -> None: + pass + + def evaluate(self, estimate) -> list: + return calculate_outlier_rate(estimate, self._cdf_limits[0], self._cdf_limits[1]) + + def finalize(self) -> None: + pass + + +class ADMetric(DistToDistMetric): + """Class wrapper for Anderson Darling metric. + """ + + metric_name = 'ad' + metric_output_type = MetricOuputType.one_value_per_distribution + + def __init__(self, num_samples:int=100) -> None: + super().__init__() + self._num_samples = num_samples + self._random_state = None + + @property + def _random_state(self): + return self._random_state + + @_random_state.setter + def _random_state(self, random_state): + self._random_state=random_state + + def initialize(self): + pass + + def evaluate(self, estimate, reference) -> list: + return calculate_goodness_of_fit( + estimate, + reference, + fit_metric=self.metric_name, + num_samples=self._num_samples, + _random_state=self._random_state + ) + + def finalize(self) -> None: + pass + + +class CvMMetric(DistToDistMetric): + """Class wrapper for Cramer von Mises metric. + """ + + metric_name = 'cvm' + metric_output_type = MetricOuputType.one_value_per_distribution + + def __init__(self, num_samples:int=100) -> None: + super().__init__() + self._num_samples = num_samples + self._random_state = None + + @property + def _random_state(self): + return self._random_state + + @_random_state.setter + def _random_state(self, random_state): + self._random_state=random_state + + def initialize(self): + pass + + def evaluate(self, estimate, reference) -> list: + return calculate_goodness_of_fit( + estimate, + reference, + fit_metric=self.metric_name, + num_samples=self._num_samples, + _random_state=self._random_state + ) + + def finalize(self) -> None: + pass + + +class KSMetric(DistToDistMetric): + """Class wrapper for Kolmogorov Smirnov metric. + """ + + metric_name = 'ks' + metric_output_type = MetricOuputType.one_value_per_distribution + + def __init__(self, num_samples:int=100) -> None: + super().__init__() + self._num_samples = num_samples + self._random_state = None + + @property + def _random_state(self): + return self._random_state + + @_random_state.setter + def _random_state(self, random_state): + self._random_state=random_state + + def initialize(self): + pass + + def evaluate(self, estimate, reference) -> list: + return calculate_goodness_of_fit( + estimate, + reference, + fit_metric=self.metric_name, + num_samples=self._num_samples, + _random_state=self._random_state + ) + + def finalize(self) -> None: + pass