You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2025/01/24 04:45:35 UTC
(spark) branch master updated: [SPARK-50963][ML][PYTHON][TESTS][FOLLOW-UP] Enable a parity test
This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 3377962bc918 [SPARK-50963][ML][PYTHON][TESTS][FOLLOW-UP] Enable a parity test
3377962bc918 is described below
commit 3377962bc918858230eb121e752ec4eeaa3f44f0
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Fri Jan 24 12:45:16 2025 +0800
[SPARK-50963][ML][PYTHON][TESTS][FOLLOW-UP] Enable a parity test
### What changes were proposed in this pull request?
Enable a existing test on connect, move it after `test_stop_words_remover` and rename it
### Why are the changes needed?
for test coverage
### Does this PR introduce _any_ user-facing change?
no, test-only
### How was this patch tested?
parity test
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #49642 from zhengruifeng/ml_remover_test.
Authored-by: Ruifeng Zheng <ru...@apache.org>
Signed-off-by: Ruifeng Zheng <ru...@apache.org>
---
.../ml/tests/connect/test_parity_feature.py | 4 --
python/pyspark/ml/tests/test_feature.py | 62 +++++++++++-----------
2 files changed, 31 insertions(+), 35 deletions(-)
diff --git a/python/pyspark/ml/tests/connect/test_parity_feature.py b/python/pyspark/ml/tests/connect/test_parity_feature.py
index 105ba07df43b..c49faaf18e73 100644
--- a/python/pyspark/ml/tests/connect/test_parity_feature.py
+++ b/python/pyspark/ml/tests/connect/test_parity_feature.py
@@ -34,10 +34,6 @@ class FeatureParityTests(FeatureTestsMixin, ReusedConnectTestCase):
def test_ngram(self):
super().test_ngram()
- @unittest.skip("Need to support.")
- def test_stopwordsremover(self):
- super().test_stopwordsremover()
-
@unittest.skip("Need to support.")
def test_count_vectorizer_with_binary(self):
super().test_count_vectorizer_with_binary()
diff --git a/python/pyspark/ml/tests/test_feature.py b/python/pyspark/ml/tests/test_feature.py
index 316afe39d3ac..16480b515a13 100644
--- a/python/pyspark/ml/tests/test_feature.py
+++ b/python/pyspark/ml/tests/test_feature.py
@@ -517,6 +517,37 @@ class FeatureTestsMixin:
remover2 = StopWordsRemover.load(d)
self.assertEqual(str(remover), str(remover2))
+ def test_stop_words_remover_II(self):
+ dataset = self.spark.createDataFrame([Row(input=["a", "panda"])])
+ stopWordRemover = StopWordsRemover(inputCol="input", outputCol="output")
+ # Default
+ self.assertEqual(stopWordRemover.getInputCol(), "input")
+ transformedDF = stopWordRemover.transform(dataset)
+ self.assertEqual(transformedDF.head().output, ["panda"])
+ self.assertEqual(type(stopWordRemover.getStopWords()), list)
+ self.assertTrue(isinstance(stopWordRemover.getStopWords()[0], str))
+ # Custom
+ stopwords = ["panda"]
+ stopWordRemover.setStopWords(stopwords)
+ self.assertEqual(stopWordRemover.getInputCol(), "input")
+ self.assertEqual(stopWordRemover.getStopWords(), stopwords)
+ transformedDF = stopWordRemover.transform(dataset)
+ self.assertEqual(transformedDF.head().output, ["a"])
+ # with language selection
+ stopwords = StopWordsRemover.loadDefaultStopWords("turkish")
+ dataset = self.spark.createDataFrame([Row(input=["acaba", "ama", "biri"])])
+ stopWordRemover.setStopWords(stopwords)
+ self.assertEqual(stopWordRemover.getStopWords(), stopwords)
+ transformedDF = stopWordRemover.transform(dataset)
+ self.assertEqual(transformedDF.head().output, [])
+ # with locale
+ stopwords = ["BELKİ"]
+ dataset = self.spark.createDataFrame([Row(input=["belki"])])
+ stopWordRemover.setStopWords(stopwords).setLocale("tr")
+ self.assertEqual(stopWordRemover.getStopWords(), stopwords)
+ transformedDF = stopWordRemover.transform(dataset)
+ self.assertEqual(transformedDF.head().output, [])
+
def test_binarizer(self):
b0 = Binarizer()
self.assertListEqual(
@@ -570,37 +601,6 @@ class FeatureTestsMixin:
transformedDF = ngram0.transform(dataset)
self.assertEqual(transformedDF.head().output, ["a b c d", "b c d e"])
- def test_stopwordsremover(self):
- dataset = self.spark.createDataFrame([Row(input=["a", "panda"])])
- stopWordRemover = StopWordsRemover(inputCol="input", outputCol="output")
- # Default
- self.assertEqual(stopWordRemover.getInputCol(), "input")
- transformedDF = stopWordRemover.transform(dataset)
- self.assertEqual(transformedDF.head().output, ["panda"])
- self.assertEqual(type(stopWordRemover.getStopWords()), list)
- self.assertTrue(isinstance(stopWordRemover.getStopWords()[0], str))
- # Custom
- stopwords = ["panda"]
- stopWordRemover.setStopWords(stopwords)
- self.assertEqual(stopWordRemover.getInputCol(), "input")
- self.assertEqual(stopWordRemover.getStopWords(), stopwords)
- transformedDF = stopWordRemover.transform(dataset)
- self.assertEqual(transformedDF.head().output, ["a"])
- # with language selection
- stopwords = StopWordsRemover.loadDefaultStopWords("turkish")
- dataset = self.spark.createDataFrame([Row(input=["acaba", "ama", "biri"])])
- stopWordRemover.setStopWords(stopwords)
- self.assertEqual(stopWordRemover.getStopWords(), stopwords)
- transformedDF = stopWordRemover.transform(dataset)
- self.assertEqual(transformedDF.head().output, [])
- # with locale
- stopwords = ["BELKİ"]
- dataset = self.spark.createDataFrame([Row(input=["belki"])])
- stopWordRemover.setStopWords(stopwords).setLocale("tr")
- self.assertEqual(stopWordRemover.getStopWords(), stopwords)
- transformedDF = stopWordRemover.transform(dataset)
- self.assertEqual(transformedDF.head().output, [])
-
def test_count_vectorizer_with_binary(self):
dataset = self.spark.createDataFrame(
[
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org