You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by "zhengruifeng (via GitHub)" <gi...@apache.org> on 2023/06/30 05:10:47 UTC

[GitHub] [spark] zhengruifeng commented on a diff in pull request #41793: [SPARK-44250][ML][PYTHON][CONNECT] Implement classification evaluator

zhengruifeng commented on code in PR #41793:
URL: https://github.com/apache/spark/pull/41793#discussion_r1247441830


##########
python/pyspark/ml/connect/evaluation.py:
##########
@@ -48,42 +39,126 @@ def __init__(self, metricName: str, labelCol: str, predictionCol: str) -> None:
         typeConverter=TypeConverters.toString,
     )
 
-    def _get_torch_metric(self) -> Any:
-        metric_name = self.getOrDefault(self.metricName)
+    def _get_torch_metric(self):
+        raise NotImplementedError()
 
-        if metric_name == "mse":
-            return torchmetrics.MeanSquaredError()
-        if metric_name == "r2":
-            return torchmetrics.R2Score()
+    def _get_input_cols(self):
+        raise NotImplementedError()
 
-        raise ValueError(f"Unsupported regressor evaluator metric name: {metric_name}")
+    def _get_metric_update_inputs(self, dataset: "pd.DataFrame"):
+        raise NotImplementedError()
 
     def _evaluate(self, dataset: Union["DataFrame", "pd.DataFrame"]) -> float:
-        prediction_col = self.getPredictionCol()
-        label_col = self.getLabelCol()
-
+        import torch
         torch_metric = self._get_torch_metric()
 
         def local_agg_fn(pandas_df: "pd.DataFrame") -> "pd.DataFrame":
-            with torch.inference_mode():
-                preds_tensor = torch.tensor(pandas_df[prediction_col].values)
-                labels_tensor = torch.tensor(pandas_df[label_col].values)
-                torch_metric.update(preds_tensor, labels_tensor)
-                return torch_metric
+            torch_metric.update(*self._get_metric_update_inputs(pandas_df))
+            return torch_metric
 
         def merge_agg_state(state1: Any, state2: Any) -> Any:
-            with torch.inference_mode():
-                state1.merge_state([state2])
-                return state1
+            state1.merge_state([state2])
+            return state1
 
         def agg_state_to_result(state: Any) -> Any:
-            with torch.inference_mode():
-                return state.compute().item()
+            return state.compute().item()
 
         return aggregate_dataframe(
             dataset,
-            [prediction_col, label_col],
+            self._get_input_cols(),
             local_agg_fn,
             merge_agg_state,
             agg_state_to_result,
         )
+
+
+class RegressionEvaluator(_TorchMetricEvaluator, HasLabelCol, HasPredictionCol, ParamsReadWrite):
+    """
+    Evaluator for Regression, which expects input columns prediction and label.
+    Supported metrics are 'mse' and 'r2'.
+
+    .. versionadded:: 3.5.0
+    """
+
+    def _get_torch_metric(self) -> Any:
+        import torcheval.metrics as torchmetrics
+
+        metric_name = self.getOrDefault(self.metricName)
+
+        if metric_name == "mse":
+            return torchmetrics.MeanSquaredError()
+        if metric_name == "r2":
+            return torchmetrics.R2Score()
+
+        raise ValueError(f"Unsupported regressor evaluator metric name: {metric_name}")
+
+    def _get_input_cols(self):
+        return [self.getPredictionCol(), self.getLabelCol()]
+
+    def _get_metric_update_inputs(self, dataset: "pd.DataFrame"):
+        import torch
+        preds_tensor = torch.tensor(dataset[self.getPredictionCol()].values)
+        labels_tensor = torch.tensor(dataset[self.getLabelCol()].values)
+        return preds_tensor, labels_tensor
+
+
+class BinaryClassificationEvaluator(_TorchMetricEvaluator, HasLabelCol, HasProbabilityCol, ParamsReadWrite):
+    """
+    Evaluator for binary classification, which expects input columns prediction and label.
+    Supported metrics are 'areaUnderROC' and 'areaUnderPR'.
+
+    .. versionadded:: 3.5.0
+    """
+
+    def _get_torch_metric(self) -> Any:
+        import torcheval.metrics as torchmetrics
+
+        metric_name = self.getOrDefault(self.metricName)
+
+        if metric_name == "areaUnderROC":
+            return torchmetrics.BinaryAUROC()
+        if metric_name == "areaUnderPR":
+            return torchmetrics.BinaryAUPRC()
+
+        raise ValueError(f"Unsupported binary classification evaluator metric name: {metric_name}")
+
+    def _get_input_cols(self):
+        return [self.getProbabilityCol(), self.getLabelCol()]
+
+    def _get_metric_update_inputs(self, dataset: "pd.DataFrame"):
+        import torch
+        preds_tensor = torch.tensor(dataset[self.getProbabilityCol()].values)
+        if preds_tensor.dim() == 2:
+            preds_tensor = preds_tensor[:, 1]
+        labels_tensor = torch.tensor(dataset[self.getLabelCol()].values)
+        return preds_tensor, labels_tensor
+
+
+class MulticlassClassificationEvaluator(_TorchMetricEvaluator, HasLabelCol, HasPredictionCol, ParamsReadWrite):
+    """
+    Evaluator for multiclass classification, which expects input columns prediction and label.
+    Supported metrics are 'areaUnderROC' and 'areaUnderPR'.

Review Comment:
   'areaUnderROC' and 'areaUnderPR'. ?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org