You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2025/02/11 16:54:42 UTC

(spark) branch master updated: [SPARK-50940][SPARK-50941][ML][PYTHON][CONNECT][FOLLOW-UP] Directly reuse the CrossValidatorModelWriter and TrainValidationSplitModelWriter

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new c66cc0ea7dfa [SPARK-50940][SPARK-50941][ML][PYTHON][CONNECT][FOLLOW-UP] Directly reuse the CrossValidatorModelWriter and TrainValidationSplitModelWriter
c66cc0ea7dfa is described below

commit c66cc0ea7dfa0fb7e5cf353f7d151f0637fd67ec
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Tue Feb 11 08:54:28 2025 -0800

    [SPARK-50940][SPARK-50941][ML][PYTHON][CONNECT][FOLLOW-UP] Directly reuse the CrossValidatorModelWriter and TrainValidationSplitModelWriter
    
    ### What changes were proposed in this pull request?
    Directly reuse the CrossValidatorModelWriter and TrainValidationSplitModelWriter
    
    ### Why are the changes needed?
    to simplify the code, we don't need dedicated writers for connect
    
    ### Does this PR introduce _any_ user-facing change?
    no, internal changes
    
    ### How was this patch tested?
    existing tests
    
    ### Was this patch authored or co-authored using generative AI tooling?
    no
    
    Closes #49882 from zhengruifeng/ml_connect_simplify_writer.
    
    Authored-by: Ruifeng Zheng <ru...@apache.org>
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 python/pyspark/ml/connect/readwrite.py             | 60 +++++++++-------------
 .../pyspark/ml/tests/connect/test_parity_tuning.py | 21 +-------
 2 files changed, 24 insertions(+), 57 deletions(-)

diff --git a/python/pyspark/ml/connect/readwrite.py b/python/pyspark/ml/connect/readwrite.py
index c2367282b7c4..95551f67c012 100644
--- a/python/pyspark/ml/connect/readwrite.py
+++ b/python/pyspark/ml/connect/readwrite.py
@@ -15,16 +15,10 @@
 # limitations under the License.
 #
 
-from typing import cast, Type, TYPE_CHECKING, Union, Dict, Any, Optional
+from typing import cast, Type, TYPE_CHECKING, Union, Dict, Any
 
 import pyspark.sql.connect.proto as pb2
 from pyspark.ml.connect.serialize import serialize_ml_params, deserialize, deserialize_param
-from pyspark.ml.tuning import (
-    CrossValidatorModelWriter,
-    CrossValidatorModel,
-    TrainValidationSplitModel,
-    TrainValidationSplitModelWriter,
-)
 from pyspark.ml.util import MLWriter, MLReader, RL
 from pyspark.ml.wrapper import JavaWrapper
 
@@ -34,32 +28,6 @@ if TYPE_CHECKING:
     from pyspark.ml.util import JavaMLReadable, JavaMLWritable
 
 
-class RemoteCrossValidatorModelWriter(CrossValidatorModelWriter):
-    def __init__(
-        self,
-        instance: "CrossValidatorModel",
-        optionMap: Dict[str, Any] = {},
-        session: Optional["SparkSession"] = None,
-    ):
-        super(RemoteCrossValidatorModelWriter, self).__init__(instance)
-        self.instance = instance
-        self.optionMap = optionMap
-        self.session(session)  # type: ignore[arg-type]
-
-
-class RemoteTrainValidationSplitModelWriter(TrainValidationSplitModelWriter):
-    def __init__(
-        self,
-        instance: "TrainValidationSplitModel",
-        optionMap: Dict[str, Any] = {},
-        session: Optional["SparkSession"] = None,
-    ):
-        super(RemoteTrainValidationSplitModelWriter, self).__init__(instance)
-        self.instance = instance
-        self.optionMap = optionMap
-        self.session(session)  # type: ignore[arg-type]
-
-
 class RemoteMLWriter(MLWriter):
     def __init__(self, instance: "JavaMLWritable") -> None:
         super().__init__()
@@ -94,9 +62,14 @@ class RemoteMLWriter(MLWriter):
         from pyspark.ml.wrapper import JavaModel, JavaEstimator, JavaTransformer
         from pyspark.ml.evaluation import JavaEvaluator
         from pyspark.ml.pipeline import Pipeline, PipelineModel
-        from pyspark.ml.tuning import CrossValidator, TrainValidationSplit
         from pyspark.ml.classification import OneVsRest, OneVsRestModel
         from pyspark.ml.clustering import PowerIterationClustering
+        from pyspark.ml.tuning import (
+            CrossValidator,
+            CrossValidatorModel,
+            TrainValidationSplit,
+            TrainValidationSplitModel,
+        )
 
         # Spark Connect ML is built on scala Spark.ML, that means we're only
         # supporting JavaModel or JavaEstimator or JavaEvaluator
@@ -162,8 +135,12 @@ class RemoteMLWriter(MLWriter):
             cv_writer.session(session)  # type: ignore[arg-type]
             cv_writer.save(path)
         elif isinstance(instance, CrossValidatorModel):
+            from pyspark.ml.tuning import CrossValidatorModelWriter
+
             RemoteMLWriter.handleOverwrite(path, shouldOverwrite)
-            cvm_writer = RemoteCrossValidatorModelWriter(instance, optionMap, session)
+            cvm_writer = CrossValidatorModelWriter(instance)
+            cvm_writer.optionMap = optionMap
+            cvm_writer.session(session)  # type: ignore[arg-type]
             cvm_writer.save(path)
         elif isinstance(instance, TrainValidationSplit):
             from pyspark.ml.tuning import TrainValidationSplitWriter
@@ -172,8 +149,12 @@ class RemoteMLWriter(MLWriter):
             tvs_writer = TrainValidationSplitWriter(instance)
             tvs_writer.save(path)
         elif isinstance(instance, TrainValidationSplitModel):
+            from pyspark.ml.tuning import TrainValidationSplitModelWriter
+
             RemoteMLWriter.handleOverwrite(path, shouldOverwrite)
-            tvsm_writer = RemoteTrainValidationSplitModelWriter(instance, optionMap, session)
+            tvsm_writer = TrainValidationSplitModelWriter(instance)
+            tvsm_writer.optionMap = optionMap
+            tvsm_writer.session(session)  # type: ignore[arg-type]
             tvsm_writer.save(path)
         elif isinstance(instance, OneVsRest):
             from pyspark.ml.classification import OneVsRestWriter
@@ -238,9 +219,14 @@ class RemoteMLReader(MLReader[RL]):
         from pyspark.ml.wrapper import JavaModel, JavaEstimator, JavaTransformer
         from pyspark.ml.evaluation import JavaEvaluator
         from pyspark.ml.pipeline import Pipeline, PipelineModel
-        from pyspark.ml.tuning import CrossValidator, TrainValidationSplit
         from pyspark.ml.classification import OneVsRest, OneVsRestModel
         from pyspark.ml.clustering import PowerIterationClustering
+        from pyspark.ml.tuning import (
+            CrossValidator,
+            CrossValidatorModel,
+            TrainValidationSplit,
+            TrainValidationSplitModel,
+        )
 
         if (
             issubclass(clazz, JavaModel)
diff --git a/python/pyspark/ml/tests/connect/test_parity_tuning.py b/python/pyspark/ml/tests/connect/test_parity_tuning.py
index 473661d6c288..2d21644ceed5 100644
--- a/python/pyspark/ml/tests/connect/test_parity_tuning.py
+++ b/python/pyspark/ml/tests/connect/test_parity_tuning.py
@@ -17,31 +17,12 @@
 
 import unittest
 
-from pyspark.ml.classification import LogisticRegression
-from pyspark.ml.connect.readwrite import RemoteCrossValidatorModelWriter
-from pyspark.ml.linalg import Vectors
 from pyspark.ml.tests.test_tuning import TuningTestsMixin
-from pyspark.ml.tuning import CrossValidatorModel
 from pyspark.testing.connectutils import ReusedConnectTestCase
 
 
 class TuningParityTests(TuningTestsMixin, ReusedConnectTestCase):
-    def test_remote_cross_validator_model_writer(self):
-        df = self.spark.createDataFrame(
-            [
-                (1.0, 1.0, Vectors.dense(0.0, 5.0)),
-                (0.0, 2.0, Vectors.dense(1.0, 2.0)),
-                (1.0, 3.0, Vectors.dense(2.0, 1.0)),
-                (0.0, 4.0, Vectors.dense(3.0, 3.0)),
-            ],
-            ["label", "weight", "features"],
-        )
-
-        lor = LogisticRegression()
-        lor_model = lor.fit(df)
-        cv_model = CrossValidatorModel(lor_model)
-        writer = RemoteCrossValidatorModelWriter(cv_model, {"a": "b"}, self.spark)
-        self.assertEqual(writer.optionMap["a"], "b")
+    pass
 
 
 if __name__ == "__main__":


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org