From 6b35221a1739bb628ad00f2d10ff96a4c88a3093 Mon Sep 17 00:00:00 2001 From: Dmitry Khorkin <32310771+DimaAmega@users.noreply.github.com> Date: Fri, 24 Jan 2025 01:56:27 +0400 Subject: [PATCH 1/7] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8ac38d2812..296c026113 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ install_requires=[ "plotly>=5.10.0", "statsmodels>=0.12.2", - "scikit-learn>=1.0.1,<1.6.0", + "scikit-learn>=1.0.1", "pandas[parquet]>=1.3.5", "numpy>=1.22.0,<2.1", "nltk>=3.6.7", From 12211c8fbed2b6df77fb739dfaca2258ed8e20c6 Mon Sep 17 00:00:00 2001 From: DimaAmega Date: Thu, 30 Jan 2025 02:59:29 +0400 Subject: [PATCH 2/7] compat version --- .../regression_dummy_metric.py | 8 +++---- .../regression_performance_metrics.py | 13 +++++------- .../regression_quality.py | 13 +++++------- src/evidently/metrics/utils.py | 21 +++++++++++++++++++ 4 files changed, 34 insertions(+), 21 deletions(-) diff --git a/src/evidently/metrics/regression_performance/regression_dummy_metric.py b/src/evidently/metrics/regression_performance/regression_dummy_metric.py index 781a581f4c..fb31bb4e0c 100644 --- a/src/evidently/metrics/regression_performance/regression_dummy_metric.py +++ b/src/evidently/metrics/regression_performance/regression_dummy_metric.py @@ -5,12 +5,12 @@ import pandas as pd from sklearn.metrics import mean_absolute_error from sklearn.metrics import mean_absolute_percentage_error -from sklearn.metrics import mean_squared_error from evidently.base_metric import InputData from evidently.base_metric import Metric from evidently.base_metric import MetricResult from evidently.metrics.regression_performance.regression_quality import RegressionQualityMetric +from evidently.metrics.utils import root_mean_squared_error from evidently.model.widget import BaseWidgetInfo from evidently.options.base import AnyOptions from evidently.renderers.base_renderer import MetricRenderer @@ -76,10 +76,9 @@ def calculate(self, data: InputData) -> RegressionDummyMetricResults: ) # rmse dummy_preds = data.current_data[target_name].mean() - rmse_default = mean_squared_error( + rmse_default = root_mean_squared_error( y_true=data.current_data[target_name], y_pred=[dummy_preds] * data.current_data.shape[0], - squared=False, ) # mape default values # optimal constant for mape @@ -118,10 +117,9 @@ def calculate(self, data: InputData) -> RegressionDummyMetricResults: ) # rmse dummy_preds = data.reference_data[target_name].mean() - rmse_by_ref = mean_squared_error( + rmse_by_ref = root_mean_squared_error( y_true=data.current_data[target_name], y_pred=[dummy_preds] * data.current_data.shape[0], - squared=False, ) # mape default values # optimal constant for mape diff --git a/src/evidently/metrics/regression_performance/regression_performance_metrics.py b/src/evidently/metrics/regression_performance/regression_performance_metrics.py index 5f6b2d7c70..9137809ef0 100644 --- a/src/evidently/metrics/regression_performance/regression_performance_metrics.py +++ b/src/evidently/metrics/regression_performance/regression_performance_metrics.py @@ -5,7 +5,6 @@ import numpy as np from sklearn.metrics import mean_absolute_error from sklearn.metrics import mean_absolute_percentage_error -from sklearn.metrics import mean_squared_error from sklearn.metrics import r2_score from evidently.base_metric import InputData @@ -19,6 +18,7 @@ from evidently.metrics.regression_performance.objects import RegressionMetricsScatter from evidently.metrics.regression_performance.utils import apply_func_to_binned_data from evidently.metrics.utils import make_target_bins_for_reg_plots +from evidently.metrics.utils import root_mean_squared_error from evidently.model.widget import BaseWidgetInfo from evidently.renderers.base_renderer import MetricRenderer from evidently.renderers.base_renderer import default_renderer @@ -123,10 +123,9 @@ def calculate(self, data: InputData) -> RegressionPerformanceMetricsResults: y_true=data.current_data[data.column_mapping.target], y_pred=data.current_data[data.column_mapping.prediction], ) - rmse_score_value = mean_squared_error( + rmse_score_value = root_mean_squared_error( y_true=data.current_data[data.column_mapping.target], y_pred=data.current_data[data.column_mapping.prediction], - squared=False, ) # mae default values @@ -138,16 +137,14 @@ def calculate(self, data: InputData) -> RegressionPerformanceMetricsResults: # rmse default values rmse_ref = None if data.reference_data is not None: - rmse_ref = mean_squared_error( + rmse_ref = root_mean_squared_error( y_true=data.reference_data[data.column_mapping.target], y_pred=data.reference_data[data.column_mapping.prediction], - squared=False, ) dummy_preds = data.current_data[data.column_mapping.target].mean() - rmse_default = mean_squared_error( + rmse_default = root_mean_squared_error( y_true=data.current_data[data.column_mapping.target], y_pred=[dummy_preds] * data.current_data.shape[0], - squared=False, ) # mape default values # optimal constant for mape @@ -211,7 +208,7 @@ def calculate(self, data: InputData) -> RegressionPerformanceMetricsResults: ["r2_score", "rmse", "mean_abs_error", "mean_abs_perc_error"], [ r2_score, - lambda x, y: mean_squared_error(x, y, squared=False), + lambda x, y: root_mean_squared_error(x, y), mean_absolute_error, mean_absolute_percentage_error, ], diff --git a/src/evidently/metrics/regression_performance/regression_quality.py b/src/evidently/metrics/regression_performance/regression_quality.py index 7c36a8c7a2..eee9f9a139 100644 --- a/src/evidently/metrics/regression_performance/regression_quality.py +++ b/src/evidently/metrics/regression_performance/regression_quality.py @@ -6,7 +6,6 @@ import numpy as np from sklearn.metrics import mean_absolute_error from sklearn.metrics import mean_absolute_percentage_error -from sklearn.metrics import mean_squared_error from sklearn.metrics import r2_score from evidently.base_metric import InputData @@ -21,6 +20,7 @@ from evidently.metrics.regression_performance.regression_performance_metrics import RegressionMetrics from evidently.metrics.regression_performance.utils import apply_func_to_binned_data from evidently.metrics.utils import make_target_bins_for_reg_plots +from evidently.metrics.utils import root_mean_squared_error from evidently.model.widget import BaseWidgetInfo from evidently.renderers.base_renderer import MetricRenderer from evidently.renderers.base_renderer import default_renderer @@ -116,10 +116,9 @@ def calculate(self, data: InputData) -> RegressionQualityMetricResults: y_true=data.current_data[target_name], y_pred=data.current_data[prediction_name], ) - rmse_score_value = mean_squared_error( + rmse_score_value = root_mean_squared_error( y_true=data.current_data[target_name], y_pred=data.current_data[prediction_name], - squared=False, ) # mae default values @@ -131,16 +130,14 @@ def calculate(self, data: InputData) -> RegressionQualityMetricResults: # rmse default values rmse_ref = None if data.reference_data is not None: - rmse_ref = mean_squared_error( + rmse_ref = root_mean_squared_error( y_true=data.reference_data[target_name], y_pred=data.reference_data[prediction_name], - squared=False, ) dummy_preds = data.current_data[target_name].mean() - rmse_default = mean_squared_error( + rmse_default = root_mean_squared_error( y_true=data.current_data[target_name], y_pred=[dummy_preds] * data.current_data.shape[0], - squared=False, ) # mape default values # optimal constant for mape @@ -207,7 +204,7 @@ def calculate(self, data: InputData) -> RegressionQualityMetricResults: ["r2_score", "rmse", "mean_abs_error", "mean_abs_perc_error"], [ r2_score, - lambda x, y: mean_squared_error(x, y, squared=False), + lambda x, y: root_mean_squared_error(x, y), mean_absolute_error, mean_absolute_percentage_error, ], diff --git a/src/evidently/metrics/utils.py b/src/evidently/metrics/utils.py index caf07a000e..80e526fe92 100644 --- a/src/evidently/metrics/utils.py +++ b/src/evidently/metrics/utils.py @@ -1,4 +1,25 @@ import pandas as pd +import sklearn.metrics + +use_new_root_mean_squared_error = True if hasattr(sklearn.metrics, "root_mean_squared_error") else False + + +def root_mean_squared_error(y_true, y_pred): + """ + Compute the Root Mean Squared Error (RMSE) in a way that is compatible + with both old and new versions of scikit-learn. + + In scikit-learn >= 1.6.0, uses sklearn.metrics.root_mean_squared_error. + In earlier versions, uses mean_squared_error with squared=False. + """ + if use_new_root_mean_squared_error: + from sklearn.metrics import root_mean_squared_error + + return root_mean_squared_error(y_true, y_pred) + + from sklearn.metrics import mean_squared_error + + return mean_squared_error(y_true, y_pred, squared=False) def make_target_bins_for_reg_plots( From 2dff244fabbca7f48edd9cc26362a8e6725a0c0f Mon Sep 17 00:00:00 2001 From: DimaAmega Date: Thu, 30 Jan 2025 03:49:15 +0400 Subject: [PATCH 3/7] fix test --- .../classification_dummy_metric.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/evidently/metrics/classification_performance/classification_dummy_metric.py b/src/evidently/metrics/classification_performance/classification_dummy_metric.py index cdf9452b1a..8a89aded39 100644 --- a/src/evidently/metrics/classification_performance/classification_dummy_metric.py +++ b/src/evidently/metrics/classification_performance/classification_dummy_metric.py @@ -132,7 +132,11 @@ def calculate(self, data: InputData) -> ClassificationDummyMetricResults: coeff_precision = min(1.0, (1 - threshold) / 0.5) neg_label_precision = precision_score(target, dummy_preds, pos_label=labels[1]) * coeff_precision neg_label_recall = recall_score(target, dummy_preds, pos_label=labels[1]) * coeff_recall - f1_label2_value = 2 * neg_label_precision * neg_label_recall / (neg_label_precision + neg_label_recall) + f1_label2_value = ( + 2 * neg_label_precision * neg_label_recall / (neg_label_precision + neg_label_recall) + if (neg_label_precision + neg_label_recall) != 0 + else float("nan") + ) metrics_matrix = { str(labels[0]): ClassMetric( precision=current_dummy.precision, From daf472eab9ae3bb10155835f8e7e82c5d7063558 Mon Sep 17 00:00:00 2001 From: DimaAmega Date: Thu, 30 Jan 2025 04:06:09 +0400 Subject: [PATCH 4/7] rename --- .../regression_performance/regression_dummy_metric.py | 6 +++--- .../regression_performance_metrics.py | 10 +++++----- .../regression_performance/regression_quality.py | 10 +++++----- src/evidently/metrics/utils.py | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/evidently/metrics/regression_performance/regression_dummy_metric.py b/src/evidently/metrics/regression_performance/regression_dummy_metric.py index fb31bb4e0c..209f7497c8 100644 --- a/src/evidently/metrics/regression_performance/regression_dummy_metric.py +++ b/src/evidently/metrics/regression_performance/regression_dummy_metric.py @@ -10,7 +10,7 @@ from evidently.base_metric import Metric from evidently.base_metric import MetricResult from evidently.metrics.regression_performance.regression_quality import RegressionQualityMetric -from evidently.metrics.utils import root_mean_squared_error +from evidently.metrics.utils import root_mean_squared_error_compat from evidently.model.widget import BaseWidgetInfo from evidently.options.base import AnyOptions from evidently.renderers.base_renderer import MetricRenderer @@ -76,7 +76,7 @@ def calculate(self, data: InputData) -> RegressionDummyMetricResults: ) # rmse dummy_preds = data.current_data[target_name].mean() - rmse_default = root_mean_squared_error( + rmse_default = root_mean_squared_error_compat( y_true=data.current_data[target_name], y_pred=[dummy_preds] * data.current_data.shape[0], ) @@ -117,7 +117,7 @@ def calculate(self, data: InputData) -> RegressionDummyMetricResults: ) # rmse dummy_preds = data.reference_data[target_name].mean() - rmse_by_ref = root_mean_squared_error( + rmse_by_ref = root_mean_squared_error_compat( y_true=data.current_data[target_name], y_pred=[dummy_preds] * data.current_data.shape[0], ) diff --git a/src/evidently/metrics/regression_performance/regression_performance_metrics.py b/src/evidently/metrics/regression_performance/regression_performance_metrics.py index 9137809ef0..c36a1e66f0 100644 --- a/src/evidently/metrics/regression_performance/regression_performance_metrics.py +++ b/src/evidently/metrics/regression_performance/regression_performance_metrics.py @@ -18,7 +18,7 @@ from evidently.metrics.regression_performance.objects import RegressionMetricsScatter from evidently.metrics.regression_performance.utils import apply_func_to_binned_data from evidently.metrics.utils import make_target_bins_for_reg_plots -from evidently.metrics.utils import root_mean_squared_error +from evidently.metrics.utils import root_mean_squared_error_compat from evidently.model.widget import BaseWidgetInfo from evidently.renderers.base_renderer import MetricRenderer from evidently.renderers.base_renderer import default_renderer @@ -123,7 +123,7 @@ def calculate(self, data: InputData) -> RegressionPerformanceMetricsResults: y_true=data.current_data[data.column_mapping.target], y_pred=data.current_data[data.column_mapping.prediction], ) - rmse_score_value = root_mean_squared_error( + rmse_score_value = root_mean_squared_error_compat( y_true=data.current_data[data.column_mapping.target], y_pred=data.current_data[data.column_mapping.prediction], ) @@ -137,12 +137,12 @@ def calculate(self, data: InputData) -> RegressionPerformanceMetricsResults: # rmse default values rmse_ref = None if data.reference_data is not None: - rmse_ref = root_mean_squared_error( + rmse_ref = root_mean_squared_error_compat( y_true=data.reference_data[data.column_mapping.target], y_pred=data.reference_data[data.column_mapping.prediction], ) dummy_preds = data.current_data[data.column_mapping.target].mean() - rmse_default = root_mean_squared_error( + rmse_default = root_mean_squared_error_compat( y_true=data.current_data[data.column_mapping.target], y_pred=[dummy_preds] * data.current_data.shape[0], ) @@ -208,7 +208,7 @@ def calculate(self, data: InputData) -> RegressionPerformanceMetricsResults: ["r2_score", "rmse", "mean_abs_error", "mean_abs_perc_error"], [ r2_score, - lambda x, y: root_mean_squared_error(x, y), + lambda x, y: root_mean_squared_error_compat(x, y), mean_absolute_error, mean_absolute_percentage_error, ], diff --git a/src/evidently/metrics/regression_performance/regression_quality.py b/src/evidently/metrics/regression_performance/regression_quality.py index eee9f9a139..89d3182968 100644 --- a/src/evidently/metrics/regression_performance/regression_quality.py +++ b/src/evidently/metrics/regression_performance/regression_quality.py @@ -20,7 +20,7 @@ from evidently.metrics.regression_performance.regression_performance_metrics import RegressionMetrics from evidently.metrics.regression_performance.utils import apply_func_to_binned_data from evidently.metrics.utils import make_target_bins_for_reg_plots -from evidently.metrics.utils import root_mean_squared_error +from evidently.metrics.utils import root_mean_squared_error_compat from evidently.model.widget import BaseWidgetInfo from evidently.renderers.base_renderer import MetricRenderer from evidently.renderers.base_renderer import default_renderer @@ -116,7 +116,7 @@ def calculate(self, data: InputData) -> RegressionQualityMetricResults: y_true=data.current_data[target_name], y_pred=data.current_data[prediction_name], ) - rmse_score_value = root_mean_squared_error( + rmse_score_value = root_mean_squared_error_compat( y_true=data.current_data[target_name], y_pred=data.current_data[prediction_name], ) @@ -130,12 +130,12 @@ def calculate(self, data: InputData) -> RegressionQualityMetricResults: # rmse default values rmse_ref = None if data.reference_data is not None: - rmse_ref = root_mean_squared_error( + rmse_ref = root_mean_squared_error_compat( y_true=data.reference_data[target_name], y_pred=data.reference_data[prediction_name], ) dummy_preds = data.current_data[target_name].mean() - rmse_default = root_mean_squared_error( + rmse_default = root_mean_squared_error_compat( y_true=data.current_data[target_name], y_pred=[dummy_preds] * data.current_data.shape[0], ) @@ -204,7 +204,7 @@ def calculate(self, data: InputData) -> RegressionQualityMetricResults: ["r2_score", "rmse", "mean_abs_error", "mean_abs_perc_error"], [ r2_score, - lambda x, y: root_mean_squared_error(x, y), + lambda x, y: root_mean_squared_error_compat(x, y), mean_absolute_error, mean_absolute_percentage_error, ], diff --git a/src/evidently/metrics/utils.py b/src/evidently/metrics/utils.py index 80e526fe92..2bb005b9f9 100644 --- a/src/evidently/metrics/utils.py +++ b/src/evidently/metrics/utils.py @@ -4,7 +4,7 @@ use_new_root_mean_squared_error = True if hasattr(sklearn.metrics, "root_mean_squared_error") else False -def root_mean_squared_error(y_true, y_pred): +def root_mean_squared_error_compat(y_true, y_pred): """ Compute the Root Mean Squared Error (RMSE) in a way that is compatible with both old and new versions of scikit-learn. From f2511727a81e0707f60f1f50bbd6ccd47731c592 Mon Sep 17 00:00:00 2001 From: DimaAmega Date: Thu, 30 Jan 2025 04:10:14 +0400 Subject: [PATCH 5/7] fix yet another one test --- .../classification_dummy_metric.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/evidently/metrics/classification_performance/classification_dummy_metric.py b/src/evidently/metrics/classification_performance/classification_dummy_metric.py index 8a89aded39..e499ca8f87 100644 --- a/src/evidently/metrics/classification_performance/classification_dummy_metric.py +++ b/src/evidently/metrics/classification_performance/classification_dummy_metric.py @@ -246,16 +246,19 @@ def correction_for_threshold( fpr = dummy_results.fpr * coeff_recall fnr = dummy_results.fnr * coeff_precision + f1_denominator = dummy_results.precision * coeff_precision + dummy_results.recall * coeff_recall + + f1 = ( + 2 * dummy_results.precision * coeff_precision * dummy_results.recall * coeff_recall / f1_denominator + if f1_denominator != 0 + else float("nan") + ) + return DatasetClassificationQuality( accuracy=dummy_results.accuracy, precision=dummy_results.precision * coeff_precision, recall=dummy_results.recall * coeff_recall, - f1=2 - * dummy_results.precision - * coeff_precision - * dummy_results.recall - * coeff_recall - / (dummy_results.precision * coeff_precision + dummy_results.recall * coeff_recall), + f1=f1, roc_auc=0.5, log_loss=None, tpr=tpr, From e28f5a8401e6d90e00075d62efbf21583a1031a7 Mon Sep 17 00:00:00 2001 From: Dmitry Khorkin <32310771+DimaAmega@users.noreply.github.com> Date: Thu, 30 Jan 2025 16:53:02 +0400 Subject: [PATCH 6/7] Update src/evidently/metrics/utils.py Co-authored-by: Mikhail Sveshnikov --- src/evidently/metrics/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evidently/metrics/utils.py b/src/evidently/metrics/utils.py index 2bb005b9f9..bfce13fe85 100644 --- a/src/evidently/metrics/utils.py +++ b/src/evidently/metrics/utils.py @@ -1,7 +1,7 @@ import pandas as pd import sklearn.metrics -use_new_root_mean_squared_error = True if hasattr(sklearn.metrics, "root_mean_squared_error") else False +use_new_root_mean_squared_error = hasattr(sklearn.metrics, "root_mean_squared_error") def root_mean_squared_error_compat(y_true, y_pred): From 67378eddb681ceaab0dc411ac4ea559adb011b8f Mon Sep 17 00:00:00 2001 From: DimaAmega Date: Thu, 30 Jan 2025 17:05:09 +0400 Subject: [PATCH 7/7] bit refactor --- .../classification_dummy_metric.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/evidently/metrics/classification_performance/classification_dummy_metric.py b/src/evidently/metrics/classification_performance/classification_dummy_metric.py index e499ca8f87..bf64666a32 100644 --- a/src/evidently/metrics/classification_performance/classification_dummy_metric.py +++ b/src/evidently/metrics/classification_performance/classification_dummy_metric.py @@ -132,9 +132,10 @@ def calculate(self, data: InputData) -> ClassificationDummyMetricResults: coeff_precision = min(1.0, (1 - threshold) / 0.5) neg_label_precision = precision_score(target, dummy_preds, pos_label=labels[1]) * coeff_precision neg_label_recall = recall_score(target, dummy_preds, pos_label=labels[1]) * coeff_recall + f1_label2_denominator = neg_label_precision + neg_label_recall f1_label2_value = ( - 2 * neg_label_precision * neg_label_recall / (neg_label_precision + neg_label_recall) - if (neg_label_precision + neg_label_recall) != 0 + 2 * neg_label_precision * neg_label_recall / f1_label2_denominator + if f1_label2_denominator != 0 else float("nan") ) metrics_matrix = {