You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2025/02/11 16:54:42 UTC
(spark) branch master updated: [SPARK-50940][SPARK-50941][ML][PYTHON][CONNECT][FOLLOW-UP] Directly reuse the CrossValidatorModelWriter and TrainValidationSplitModelWriter
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new c66cc0ea7dfa [SPARK-50940][SPARK-50941][ML][PYTHON][CONNECT][FOLLOW-UP] Directly reuse the CrossValidatorModelWriter and TrainValidationSplitModelWriter
c66cc0ea7dfa is described below
commit c66cc0ea7dfa0fb7e5cf353f7d151f0637fd67ec
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Tue Feb 11 08:54:28 2025 -0800
[SPARK-50940][SPARK-50941][ML][PYTHON][CONNECT][FOLLOW-UP] Directly reuse the CrossValidatorModelWriter and TrainValidationSplitModelWriter
### What changes were proposed in this pull request?
Directly reuse the CrossValidatorModelWriter and TrainValidationSplitModelWriter
### Why are the changes needed?
to simplify the code, we don't need dedicated writers for connect
### Does this PR introduce _any_ user-facing change?
no, internal changes
### How was this patch tested?
existing tests
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #49882 from zhengruifeng/ml_connect_simplify_writer.
Authored-by: Ruifeng Zheng <ru...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
python/pyspark/ml/connect/readwrite.py | 60 +++++++++-------------
.../pyspark/ml/tests/connect/test_parity_tuning.py | 21 +-------
2 files changed, 24 insertions(+), 57 deletions(-)
diff --git a/python/pyspark/ml/connect/readwrite.py b/python/pyspark/ml/connect/readwrite.py
index c2367282b7c4..95551f67c012 100644
--- a/python/pyspark/ml/connect/readwrite.py
+++ b/python/pyspark/ml/connect/readwrite.py
@@ -15,16 +15,10 @@
# limitations under the License.
#
-from typing import cast, Type, TYPE_CHECKING, Union, Dict, Any, Optional
+from typing import cast, Type, TYPE_CHECKING, Union, Dict, Any
import pyspark.sql.connect.proto as pb2
from pyspark.ml.connect.serialize import serialize_ml_params, deserialize, deserialize_param
-from pyspark.ml.tuning import (
- CrossValidatorModelWriter,
- CrossValidatorModel,
- TrainValidationSplitModel,
- TrainValidationSplitModelWriter,
-)
from pyspark.ml.util import MLWriter, MLReader, RL
from pyspark.ml.wrapper import JavaWrapper
@@ -34,32 +28,6 @@ if TYPE_CHECKING:
from pyspark.ml.util import JavaMLReadable, JavaMLWritable
-class RemoteCrossValidatorModelWriter(CrossValidatorModelWriter):
- def __init__(
- self,
- instance: "CrossValidatorModel",
- optionMap: Dict[str, Any] = {},
- session: Optional["SparkSession"] = None,
- ):
- super(RemoteCrossValidatorModelWriter, self).__init__(instance)
- self.instance = instance
- self.optionMap = optionMap
- self.session(session) # type: ignore[arg-type]
-
-
-class RemoteTrainValidationSplitModelWriter(TrainValidationSplitModelWriter):
- def __init__(
- self,
- instance: "TrainValidationSplitModel",
- optionMap: Dict[str, Any] = {},
- session: Optional["SparkSession"] = None,
- ):
- super(RemoteTrainValidationSplitModelWriter, self).__init__(instance)
- self.instance = instance
- self.optionMap = optionMap
- self.session(session) # type: ignore[arg-type]
-
-
class RemoteMLWriter(MLWriter):
def __init__(self, instance: "JavaMLWritable") -> None:
super().__init__()
@@ -94,9 +62,14 @@ class RemoteMLWriter(MLWriter):
from pyspark.ml.wrapper import JavaModel, JavaEstimator, JavaTransformer
from pyspark.ml.evaluation import JavaEvaluator
from pyspark.ml.pipeline import Pipeline, PipelineModel
- from pyspark.ml.tuning import CrossValidator, TrainValidationSplit
from pyspark.ml.classification import OneVsRest, OneVsRestModel
from pyspark.ml.clustering import PowerIterationClustering
+ from pyspark.ml.tuning import (
+ CrossValidator,
+ CrossValidatorModel,
+ TrainValidationSplit,
+ TrainValidationSplitModel,
+ )
# Spark Connect ML is built on scala Spark.ML, that means we're only
# supporting JavaModel or JavaEstimator or JavaEvaluator
@@ -162,8 +135,12 @@ class RemoteMLWriter(MLWriter):
cv_writer.session(session) # type: ignore[arg-type]
cv_writer.save(path)
elif isinstance(instance, CrossValidatorModel):
+ from pyspark.ml.tuning import CrossValidatorModelWriter
+
RemoteMLWriter.handleOverwrite(path, shouldOverwrite)
- cvm_writer = RemoteCrossValidatorModelWriter(instance, optionMap, session)
+ cvm_writer = CrossValidatorModelWriter(instance)
+ cvm_writer.optionMap = optionMap
+ cvm_writer.session(session) # type: ignore[arg-type]
cvm_writer.save(path)
elif isinstance(instance, TrainValidationSplit):
from pyspark.ml.tuning import TrainValidationSplitWriter
@@ -172,8 +149,12 @@ class RemoteMLWriter(MLWriter):
tvs_writer = TrainValidationSplitWriter(instance)
tvs_writer.save(path)
elif isinstance(instance, TrainValidationSplitModel):
+ from pyspark.ml.tuning import TrainValidationSplitModelWriter
+
RemoteMLWriter.handleOverwrite(path, shouldOverwrite)
- tvsm_writer = RemoteTrainValidationSplitModelWriter(instance, optionMap, session)
+ tvsm_writer = TrainValidationSplitModelWriter(instance)
+ tvsm_writer.optionMap = optionMap
+ tvsm_writer.session(session) # type: ignore[arg-type]
tvsm_writer.save(path)
elif isinstance(instance, OneVsRest):
from pyspark.ml.classification import OneVsRestWriter
@@ -238,9 +219,14 @@ class RemoteMLReader(MLReader[RL]):
from pyspark.ml.wrapper import JavaModel, JavaEstimator, JavaTransformer
from pyspark.ml.evaluation import JavaEvaluator
from pyspark.ml.pipeline import Pipeline, PipelineModel
- from pyspark.ml.tuning import CrossValidator, TrainValidationSplit
from pyspark.ml.classification import OneVsRest, OneVsRestModel
from pyspark.ml.clustering import PowerIterationClustering
+ from pyspark.ml.tuning import (
+ CrossValidator,
+ CrossValidatorModel,
+ TrainValidationSplit,
+ TrainValidationSplitModel,
+ )
if (
issubclass(clazz, JavaModel)
diff --git a/python/pyspark/ml/tests/connect/test_parity_tuning.py b/python/pyspark/ml/tests/connect/test_parity_tuning.py
index 473661d6c288..2d21644ceed5 100644
--- a/python/pyspark/ml/tests/connect/test_parity_tuning.py
+++ b/python/pyspark/ml/tests/connect/test_parity_tuning.py
@@ -17,31 +17,12 @@
import unittest
-from pyspark.ml.classification import LogisticRegression
-from pyspark.ml.connect.readwrite import RemoteCrossValidatorModelWriter
-from pyspark.ml.linalg import Vectors
from pyspark.ml.tests.test_tuning import TuningTestsMixin
-from pyspark.ml.tuning import CrossValidatorModel
from pyspark.testing.connectutils import ReusedConnectTestCase
class TuningParityTests(TuningTestsMixin, ReusedConnectTestCase):
- def test_remote_cross_validator_model_writer(self):
- df = self.spark.createDataFrame(
- [
- (1.0, 1.0, Vectors.dense(0.0, 5.0)),
- (0.0, 2.0, Vectors.dense(1.0, 2.0)),
- (1.0, 3.0, Vectors.dense(2.0, 1.0)),
- (0.0, 4.0, Vectors.dense(3.0, 3.0)),
- ],
- ["label", "weight", "features"],
- )
-
- lor = LogisticRegression()
- lor_model = lor.fit(df)
- cv_model = CrossValidatorModel(lor_model)
- writer = RemoteCrossValidatorModelWriter(cv_model, {"a": "b"}, self.spark)
- self.assertEqual(writer.optionMap["a"], "b")
+ pass
if __name__ == "__main__":
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org