You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2022/05/25 07:33:32 UTC
[spark] branch branch-3.1 updated (ad42284f3f1 -> 3cc5d4a6138)
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a change to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git
from ad42284f3f1 [SPARK-39252][PYSPARK][TESTS] Remove flaky test_df_is_empty
new 661cc5298c3 Revert "[SPARK-39252][PYSPARK][TESTS] Remove flaky test_df_is_empty"
new 3cc5d4a6138 [SPARK-39252][PYSPARK][TESTS] Remove flaky test_df_is_empty
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
python/pyspark/sql/tests/test_dataframe.py | 1 -
1 file changed, 1 deletion(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[spark] 01/02: Revert "[SPARK-39252][PYSPARK][TESTS] Remove flaky test_df_is_empty"
Posted by gu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git
commit 661cc5298c3a46e7381b1159eb90764ad616c87a
Author: Hyukjin Kwon <gu...@apache.org>
AuthorDate: Wed May 25 16:32:41 2022 +0900
Revert "[SPARK-39252][PYSPARK][TESTS] Remove flaky test_df_is_empty"
This reverts commit ad42284f3f1aa3d3d691b95ea76ea8eae535abe2.
---
python/pyspark/sql/tests/test_dataframe.py | 37 +++++++++++++++++++++++++++++-
1 file changed, 36 insertions(+), 1 deletion(-)
diff --git a/python/pyspark/sql/tests/test_dataframe.py b/python/pyspark/sql/tests/test_dataframe.py
index 562e5bbda08..6b9ac24d8c1 100644
--- a/python/pyspark/sql/tests/test_dataframe.py
+++ b/python/pyspark/sql/tests/test_dataframe.py
@@ -21,7 +21,7 @@ import shutil
import tempfile
import time
import unittest
-from typing import cast
+import uuid
from pyspark.sql import SparkSession, Row
from pyspark.sql.types import StringType, IntegerType, DoubleType, StructType, StructField, \
@@ -838,6 +838,41 @@ class DataFrameTests(ReusedSQLTestCase):
finally:
shutil.rmtree(tpath)
+ def test_df_is_empty(self):
+ # SPARK-39084: Fix df.rdd.isEmpty() resulting in JVM crash.
+
+ # This particular example of DataFrame reproduces an issue in isEmpty call
+ # which could result in JVM crash.
+ data = []
+ for t in range(0, 10000):
+ id = str(uuid.uuid4())
+ if t == 0:
+ for i in range(0, 99):
+ data.append((id,))
+ elif t < 10:
+ for i in range(0, 75):
+ data.append((id,))
+ elif t < 100:
+ for i in range(0, 50):
+ data.append((id,))
+ elif t < 1000:
+ for i in range(0, 25):
+ data.append((id,))
+ else:
+ for i in range(0, 10):
+ data.append((id,))
+
+ tmpPath = tempfile.mkdtemp()
+ shutil.rmtree(tmpPath)
+ try:
+ df = self.spark.createDataFrame(data, ["col"])
+ df.coalesce(1).write.parquet(tmpPath)
+
+ res = self.spark.read.parquet(tmpPath).groupBy("col").count()
+ self.assertFalse(res.rdd.isEmpty())
+ finally:
+ shutil.rmtree(tmpPath)
+
class QueryExecutionListenerTests(unittest.TestCase, SQLTestUtils):
# These tests are separate because it uses 'spark.sql.queryExecutionListeners' which is
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[spark] 02/02: [SPARK-39252][PYSPARK][TESTS] Remove flaky test_df_is_empty
Posted by gu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git
commit 3cc5d4a6138c58b1d290748304156dcf2fd63b6d
Author: Ivan Sadikov <iv...@databricks.com>
AuthorDate: Wed May 25 11:39:54 2022 +0900
[SPARK-39252][PYSPARK][TESTS] Remove flaky test_df_is_empty
This PR removes flaky `test_df_is_empty` as reported in https://issues.apache.org/jira/browse/SPARK-39252. I will open a follow-up PR to reintroduce the test and fix the flakiness (or see if it was a regression).
No.
Existing unit tests.
Closes #36656 from sadikovi/SPARK-39252.
Authored-by: Ivan Sadikov <iv...@databricks.com>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
(cherry picked from commit 9823bb385cd6dca7c4fb5a6315721420ad42f80a)
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
python/pyspark/sql/tests/test_dataframe.py | 36 ------------------------------
1 file changed, 36 deletions(-)
diff --git a/python/pyspark/sql/tests/test_dataframe.py b/python/pyspark/sql/tests/test_dataframe.py
index 6b9ac24d8c1..e3977e81851 100644
--- a/python/pyspark/sql/tests/test_dataframe.py
+++ b/python/pyspark/sql/tests/test_dataframe.py
@@ -21,7 +21,6 @@ import shutil
import tempfile
import time
import unittest
-import uuid
from pyspark.sql import SparkSession, Row
from pyspark.sql.types import StringType, IntegerType, DoubleType, StructType, StructField, \
@@ -838,41 +837,6 @@ class DataFrameTests(ReusedSQLTestCase):
finally:
shutil.rmtree(tpath)
- def test_df_is_empty(self):
- # SPARK-39084: Fix df.rdd.isEmpty() resulting in JVM crash.
-
- # This particular example of DataFrame reproduces an issue in isEmpty call
- # which could result in JVM crash.
- data = []
- for t in range(0, 10000):
- id = str(uuid.uuid4())
- if t == 0:
- for i in range(0, 99):
- data.append((id,))
- elif t < 10:
- for i in range(0, 75):
- data.append((id,))
- elif t < 100:
- for i in range(0, 50):
- data.append((id,))
- elif t < 1000:
- for i in range(0, 25):
- data.append((id,))
- else:
- for i in range(0, 10):
- data.append((id,))
-
- tmpPath = tempfile.mkdtemp()
- shutil.rmtree(tmpPath)
- try:
- df = self.spark.createDataFrame(data, ["col"])
- df.coalesce(1).write.parquet(tmpPath)
-
- res = self.spark.read.parquet(tmpPath).groupBy("col").count()
- self.assertFalse(res.rdd.isEmpty())
- finally:
- shutil.rmtree(tmpPath)
-
class QueryExecutionListenerTests(unittest.TestCase, SQLTestUtils):
# These tests are separate because it uses 'spark.sql.queryExecutionListeners' which is
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org