You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2018/11/15 04:31:03 UTC
[4/4] spark git commit: [SPARK-26036][PYTHON] Break large tests.py
files into smaller files
[SPARK-26036][PYTHON] Break large tests.py files into smaller files
## What changes were proposed in this pull request?
This PR continues to break down a big large file into smaller files. See https://github.com/apache/spark/pull/23021. It targets to follow https://github.com/numpy/numpy/tree/master/numpy.
Basically this PR proposes to break down `pyspark/tests.py` into ...:
```
pyspark
...
├── testing
...
│ └── utils.py
├── tests
│ ├── __init__.py
│ ├── test_appsubmit.py
│ ├── test_broadcast.py
│ ├── test_conf.py
│ ├── test_context.py
│ ├── test_daemon.py
│ ├── test_join.py
│ ├── test_profiler.py
│ ├── test_rdd.py
│ ├── test_readwrite.py
│ ├── test_serializers.py
│ ├── test_shuffle.py
│ ├── test_taskcontext.py
│ ├── test_util.py
│ └── test_worker.py
...
```
## How was this patch tested?
Existing tests should cover.
`cd python` and .`/run-tests-with-coverage`. Manually checked they are actually being ran.
Each test (not officially) can be ran via:
```bash
SPARK_TESTING=1 ./bin/pyspark pyspark.tests.test_context
```
Note that if you're using Mac and Python 3, you might have to `OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES`.
Closes #23033 from HyukjinKwon/SPARK-26036.
Authored-by: hyukjinkwon <gu...@apache.org>
Signed-off-by: hyukjinkwon <gu...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/03306a6d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/03306a6d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/03306a6d
Branch: refs/heads/master
Commit: 03306a6df39c9fd6cb581401c13c4dfc6bbd632e
Parents: f6255d7
Author: hyukjinkwon <gu...@apache.org>
Authored: Thu Nov 15 12:30:52 2018 +0800
Committer: hyukjinkwon <gu...@apache.org>
Committed: Thu Nov 15 12:30:52 2018 +0800
----------------------------------------------------------------------
dev/sparktestsupport/modules.py | 19 +-
python/pyspark/ml/tests.py | 2 +-
python/pyspark/sql/tests/test_appsubmit.py | 7 +-
python/pyspark/sql/tests/test_arrow.py | 7 +-
python/pyspark/sql/tests/test_catalog.py | 5 +-
python/pyspark/sql/tests/test_column.py | 5 +-
python/pyspark/sql/tests/test_conf.py | 5 +-
python/pyspark/sql/tests/test_context.py | 7 +-
python/pyspark/sql/tests/test_dataframe.py | 7 +-
python/pyspark/sql/tests/test_datasources.py | 5 +-
python/pyspark/sql/tests/test_functions.py | 5 +-
python/pyspark/sql/tests/test_group.py | 5 +-
python/pyspark/sql/tests/test_pandas_udf.py | 7 +-
.../sql/tests/test_pandas_udf_grouped_agg.py | 7 +-
.../sql/tests/test_pandas_udf_grouped_map.py | 7 +-
.../pyspark/sql/tests/test_pandas_udf_scalar.py | 7 +-
.../pyspark/sql/tests/test_pandas_udf_window.py | 7 +-
python/pyspark/sql/tests/test_readwriter.py | 5 +-
python/pyspark/sql/tests/test_serde.py | 5 +-
python/pyspark/sql/tests/test_session.py | 7 +-
python/pyspark/sql/tests/test_streaming.py | 5 +-
python/pyspark/sql/tests/test_types.py | 5 +-
python/pyspark/sql/tests/test_udf.py | 7 +-
python/pyspark/sql/tests/test_utils.py | 5 +-
python/pyspark/test_broadcast.py | 126 -
python/pyspark/test_serializers.py | 90 -
python/pyspark/testing/sqlutils.py | 2 +-
python/pyspark/testing/utils.py | 102 +
python/pyspark/tests.py | 2502 ------------------
python/pyspark/tests/__init__.py | 16 +
python/pyspark/tests/test_appsubmit.py | 248 ++
python/pyspark/tests/test_broadcast.py | 122 +
python/pyspark/tests/test_conf.py | 43 +
python/pyspark/tests/test_context.py | 258 ++
python/pyspark/tests/test_daemon.py | 80 +
python/pyspark/tests/test_join.py | 69 +
python/pyspark/tests/test_profiler.py | 112 +
python/pyspark/tests/test_rdd.py | 739 ++++++
python/pyspark/tests/test_readwrite.py | 499 ++++
python/pyspark/tests/test_serializers.py | 237 ++
python/pyspark/tests/test_shuffle.py | 181 ++
python/pyspark/tests/test_taskcontext.py | 161 ++
python/pyspark/tests/test_util.py | 86 +
python/pyspark/tests/test_worker.py | 157 ++
44 files changed, 3205 insertions(+), 2778 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/dev/sparktestsupport/modules.py
----------------------------------------------------------------------
diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index 9dbe4e4..d5fcc06 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -310,6 +310,7 @@ pyspark_core = Module(
"python/(?!pyspark/(ml|mllib|sql|streaming))"
],
python_test_goals=[
+ # doctests
"pyspark.rdd",
"pyspark.context",
"pyspark.conf",
@@ -318,10 +319,22 @@ pyspark_core = Module(
"pyspark.serializers",
"pyspark.profiler",
"pyspark.shuffle",
- "pyspark.tests",
- "pyspark.test_broadcast",
- "pyspark.test_serializers",
"pyspark.util",
+ # unittests
+ "pyspark.tests.test_appsubmit",
+ "pyspark.tests.test_broadcast",
+ "pyspark.tests.test_conf",
+ "pyspark.tests.test_context",
+ "pyspark.tests.test_daemon",
+ "pyspark.tests.test_join",
+ "pyspark.tests.test_profiler",
+ "pyspark.tests.test_rdd",
+ "pyspark.tests.test_readwrite",
+ "pyspark.tests.test_serializers",
+ "pyspark.tests.test_shuffle",
+ "pyspark.tests.test_taskcontext",
+ "pyspark.tests.test_util",
+ "pyspark.tests.test_worker",
]
)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/ml/tests.py
----------------------------------------------------------------------
diff --git a/python/pyspark/ml/tests.py b/python/pyspark/ml/tests.py
index 821e037..2b4b731 100755
--- a/python/pyspark/ml/tests.py
+++ b/python/pyspark/ml/tests.py
@@ -72,7 +72,7 @@ from pyspark.sql import DataFrame, Row, SparkSession, HiveContext
from pyspark.sql.functions import rand
from pyspark.sql.types import DoubleType, IntegerType
from pyspark.storagelevel import *
-from pyspark.tests import QuietTest, ReusedPySparkTestCase as PySparkTestCase
+from pyspark.testing.utils import QuietTest, ReusedPySparkTestCase as PySparkTestCase
ser = PickleSerializer()
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/sql/tests/test_appsubmit.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests/test_appsubmit.py b/python/pyspark/sql/tests/test_appsubmit.py
index 3c71151..43abcde 100644
--- a/python/pyspark/sql/tests/test_appsubmit.py
+++ b/python/pyspark/sql/tests/test_appsubmit.py
@@ -22,7 +22,7 @@ import tempfile
import py4j
from pyspark import SparkContext
-from pyspark.tests import SparkSubmitTests
+from pyspark.tests.test_appsubmit import SparkSubmitTests
class HiveSparkSubmitTests(SparkSubmitTests):
@@ -91,6 +91,7 @@ if __name__ == "__main__":
try:
import xmlrunner
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ testRunner = xmlrunner.XMLTestRunner(output='target/test-reports')
except ImportError:
- unittest.main(verbosity=2)
+ testRunner = None
+ unittest.main(testRunner=testRunner, verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/sql/tests/test_arrow.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests/test_arrow.py b/python/pyspark/sql/tests/test_arrow.py
index 44f7035..6e75e82 100644
--- a/python/pyspark/sql/tests/test_arrow.py
+++ b/python/pyspark/sql/tests/test_arrow.py
@@ -26,7 +26,7 @@ from pyspark.sql import Row
from pyspark.sql.types import *
from pyspark.testing.sqlutils import ReusedSQLTestCase, have_pandas, have_pyarrow, \
pandas_requirement_message, pyarrow_requirement_message
-from pyspark.tests import QuietTest
+from pyspark.testing.utils import QuietTest
from pyspark.util import _exception_message
@@ -394,6 +394,7 @@ if __name__ == "__main__":
try:
import xmlrunner
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ testRunner = xmlrunner.XMLTestRunner(output='target/test-reports')
except ImportError:
- unittest.main(verbosity=2)
+ testRunner = None
+ unittest.main(testRunner=testRunner, verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/sql/tests/test_catalog.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests/test_catalog.py b/python/pyspark/sql/tests/test_catalog.py
index 23d2577..873405a 100644
--- a/python/pyspark/sql/tests/test_catalog.py
+++ b/python/pyspark/sql/tests/test_catalog.py
@@ -194,6 +194,7 @@ if __name__ == "__main__":
try:
import xmlrunner
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ testRunner = xmlrunner.XMLTestRunner(output='target/test-reports')
except ImportError:
- unittest.main(verbosity=2)
+ testRunner = None
+ unittest.main(testRunner=testRunner, verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/sql/tests/test_column.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests/test_column.py b/python/pyspark/sql/tests/test_column.py
index faadde9..01d4f7e 100644
--- a/python/pyspark/sql/tests/test_column.py
+++ b/python/pyspark/sql/tests/test_column.py
@@ -152,6 +152,7 @@ if __name__ == "__main__":
try:
import xmlrunner
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ testRunner = xmlrunner.XMLTestRunner(output='target/test-reports')
except ImportError:
- unittest.main(verbosity=2)
+ testRunner = None
+ unittest.main(testRunner=testRunner, verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/sql/tests/test_conf.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests/test_conf.py b/python/pyspark/sql/tests/test_conf.py
index f5d68a8..53ac4a6 100644
--- a/python/pyspark/sql/tests/test_conf.py
+++ b/python/pyspark/sql/tests/test_conf.py
@@ -50,6 +50,7 @@ if __name__ == "__main__":
try:
import xmlrunner
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ testRunner = xmlrunner.XMLTestRunner(output='target/test-reports')
except ImportError:
- unittest.main(verbosity=2)
+ testRunner = None
+ unittest.main(testRunner=testRunner, verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/sql/tests/test_context.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests/test_context.py b/python/pyspark/sql/tests/test_context.py
index d9d408a..918f4ad 100644
--- a/python/pyspark/sql/tests/test_context.py
+++ b/python/pyspark/sql/tests/test_context.py
@@ -25,7 +25,7 @@ import py4j
from pyspark import HiveContext, Row
from pyspark.sql.types import *
from pyspark.sql.window import Window
-from pyspark.tests import ReusedPySparkTestCase
+from pyspark.testing.utils import ReusedPySparkTestCase
class HiveContextSQLTests(ReusedPySparkTestCase):
@@ -258,6 +258,7 @@ if __name__ == "__main__":
try:
import xmlrunner
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ testRunner = xmlrunner.XMLTestRunner(output='target/test-reports')
except ImportError:
- unittest.main(verbosity=2)
+ testRunner = None
+ unittest.main(testRunner=testRunner, verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/sql/tests/test_dataframe.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests/test_dataframe.py b/python/pyspark/sql/tests/test_dataframe.py
index eba00b5..908d400 100644
--- a/python/pyspark/sql/tests/test_dataframe.py
+++ b/python/pyspark/sql/tests/test_dataframe.py
@@ -25,7 +25,7 @@ from pyspark.sql.types import *
from pyspark.sql.utils import AnalysisException, IllegalArgumentException
from pyspark.testing.sqlutils import ReusedSQLTestCase, SQLTestUtils, have_pyarrow, have_pandas, \
pandas_requirement_message, pyarrow_requirement_message
-from pyspark.tests import QuietTest
+from pyspark.testing.utils import QuietTest
class DataFrameTests(ReusedSQLTestCase):
@@ -732,6 +732,7 @@ if __name__ == "__main__":
try:
import xmlrunner
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ testRunner = xmlrunner.XMLTestRunner(output='target/test-reports')
except ImportError:
- unittest.main(verbosity=2)
+ testRunner = None
+ unittest.main(testRunner=testRunner, verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/sql/tests/test_datasources.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests/test_datasources.py b/python/pyspark/sql/tests/test_datasources.py
index b827378..5579620 100644
--- a/python/pyspark/sql/tests/test_datasources.py
+++ b/python/pyspark/sql/tests/test_datasources.py
@@ -165,6 +165,7 @@ if __name__ == "__main__":
try:
import xmlrunner
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ testRunner = xmlrunner.XMLTestRunner(output='target/test-reports')
except ImportError:
- unittest.main(verbosity=2)
+ testRunner = None
+ unittest.main(testRunner=testRunner, verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/sql/tests/test_functions.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests/test_functions.py b/python/pyspark/sql/tests/test_functions.py
index f0b59e8..fe66602 100644
--- a/python/pyspark/sql/tests/test_functions.py
+++ b/python/pyspark/sql/tests/test_functions.py
@@ -273,6 +273,7 @@ if __name__ == "__main__":
try:
import xmlrunner
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ testRunner = xmlrunner.XMLTestRunner(output='target/test-reports')
except ImportError:
- unittest.main(verbosity=2)
+ testRunner = None
+ unittest.main(testRunner=testRunner, verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/sql/tests/test_group.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests/test_group.py b/python/pyspark/sql/tests/test_group.py
index 076899f..6de1b8e 100644
--- a/python/pyspark/sql/tests/test_group.py
+++ b/python/pyspark/sql/tests/test_group.py
@@ -40,6 +40,7 @@ if __name__ == "__main__":
try:
import xmlrunner
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ testRunner = xmlrunner.XMLTestRunner(output='target/test-reports')
except ImportError:
- unittest.main(verbosity=2)
+ testRunner = None
+ unittest.main(testRunner=testRunner, verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/sql/tests/test_pandas_udf.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests/test_pandas_udf.py b/python/pyspark/sql/tests/test_pandas_udf.py
index 54a34a7..c4b5478 100644
--- a/python/pyspark/sql/tests/test_pandas_udf.py
+++ b/python/pyspark/sql/tests/test_pandas_udf.py
@@ -21,7 +21,7 @@ from pyspark.sql.types import *
from pyspark.sql.utils import ParseException
from pyspark.testing.sqlutils import ReusedSQLTestCase, have_pandas, have_pyarrow, \
pandas_requirement_message, pyarrow_requirement_message
-from pyspark.tests import QuietTest
+from pyspark.testing.utils import QuietTest
@unittest.skipIf(
@@ -211,6 +211,7 @@ if __name__ == "__main__":
try:
import xmlrunner
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ testRunner = xmlrunner.XMLTestRunner(output='target/test-reports')
except ImportError:
- unittest.main(verbosity=2)
+ testRunner = None
+ unittest.main(testRunner=testRunner, verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/sql/tests/test_pandas_udf_grouped_agg.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests/test_pandas_udf_grouped_agg.py b/python/pyspark/sql/tests/test_pandas_udf_grouped_agg.py
index bca47cc..5383704 100644
--- a/python/pyspark/sql/tests/test_pandas_udf_grouped_agg.py
+++ b/python/pyspark/sql/tests/test_pandas_udf_grouped_agg.py
@@ -21,7 +21,7 @@ from pyspark.sql.types import *
from pyspark.sql.utils import AnalysisException
from pyspark.testing.sqlutils import ReusedSQLTestCase, have_pandas, have_pyarrow, \
pandas_requirement_message, pyarrow_requirement_message
-from pyspark.tests import QuietTest
+from pyspark.testing.utils import QuietTest
@unittest.skipIf(
@@ -498,6 +498,7 @@ if __name__ == "__main__":
try:
import xmlrunner
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ testRunner = xmlrunner.XMLTestRunner(output='target/test-reports')
except ImportError:
- unittest.main(verbosity=2)
+ testRunner = None
+ unittest.main(testRunner=testRunner, verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/sql/tests/test_pandas_udf_grouped_map.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests/test_pandas_udf_grouped_map.py b/python/pyspark/sql/tests/test_pandas_udf_grouped_map.py
index 4d44388..bfecc07 100644
--- a/python/pyspark/sql/tests/test_pandas_udf_grouped_map.py
+++ b/python/pyspark/sql/tests/test_pandas_udf_grouped_map.py
@@ -22,7 +22,7 @@ from pyspark.sql import Row
from pyspark.sql.types import *
from pyspark.testing.sqlutils import ReusedSQLTestCase, have_pandas, have_pyarrow, \
pandas_requirement_message, pyarrow_requirement_message
-from pyspark.tests import QuietTest
+from pyspark.testing.utils import QuietTest
@unittest.skipIf(
@@ -525,6 +525,7 @@ if __name__ == "__main__":
try:
import xmlrunner
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ testRunner = xmlrunner.XMLTestRunner(output='target/test-reports')
except ImportError:
- unittest.main(verbosity=2)
+ testRunner = None
+ unittest.main(testRunner=testRunner, verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/sql/tests/test_pandas_udf_scalar.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests/test_pandas_udf_scalar.py b/python/pyspark/sql/tests/test_pandas_udf_scalar.py
index 394ee97..2f585a3 100644
--- a/python/pyspark/sql/tests/test_pandas_udf_scalar.py
+++ b/python/pyspark/sql/tests/test_pandas_udf_scalar.py
@@ -28,7 +28,7 @@ from pyspark.sql.utils import AnalysisException
from pyspark.testing.sqlutils import ReusedSQLTestCase, test_compiled,\
test_not_compiled_message, have_pandas, have_pyarrow, pandas_requirement_message, \
pyarrow_requirement_message
-from pyspark.tests import QuietTest
+from pyspark.testing.utils import QuietTest
@unittest.skipIf(
@@ -802,6 +802,7 @@ if __name__ == "__main__":
try:
import xmlrunner
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ testRunner = xmlrunner.XMLTestRunner(output='target/test-reports')
except ImportError:
- unittest.main(verbosity=2)
+ testRunner = None
+ unittest.main(testRunner=testRunner, verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/sql/tests/test_pandas_udf_window.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests/test_pandas_udf_window.py b/python/pyspark/sql/tests/test_pandas_udf_window.py
index 26e7993..f0e6d26 100644
--- a/python/pyspark/sql/tests/test_pandas_udf_window.py
+++ b/python/pyspark/sql/tests/test_pandas_udf_window.py
@@ -21,7 +21,7 @@ from pyspark.sql.utils import AnalysisException
from pyspark.sql.window import Window
from pyspark.testing.sqlutils import ReusedSQLTestCase, have_pandas, have_pyarrow, \
pandas_requirement_message, pyarrow_requirement_message
-from pyspark.tests import QuietTest
+from pyspark.testing.utils import QuietTest
@unittest.skipIf(
@@ -257,6 +257,7 @@ if __name__ == "__main__":
try:
import xmlrunner
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ testRunner = xmlrunner.XMLTestRunner(output='target/test-reports')
except ImportError:
- unittest.main(verbosity=2)
+ testRunner = None
+ unittest.main(testRunner=testRunner, verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/sql/tests/test_readwriter.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests/test_readwriter.py b/python/pyspark/sql/tests/test_readwriter.py
index 064d308..2f8712d 100644
--- a/python/pyspark/sql/tests/test_readwriter.py
+++ b/python/pyspark/sql/tests/test_readwriter.py
@@ -148,6 +148,7 @@ if __name__ == "__main__":
try:
import xmlrunner
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ testRunner = xmlrunner.XMLTestRunner(output='target/test-reports')
except ImportError:
- unittest.main(verbosity=2)
+ testRunner = None
+ unittest.main(testRunner=testRunner, verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/sql/tests/test_serde.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests/test_serde.py b/python/pyspark/sql/tests/test_serde.py
index 5ea0636..8707f46 100644
--- a/python/pyspark/sql/tests/test_serde.py
+++ b/python/pyspark/sql/tests/test_serde.py
@@ -133,6 +133,7 @@ if __name__ == "__main__":
try:
import xmlrunner
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ testRunner = xmlrunner.XMLTestRunner(output='target/test-reports')
except ImportError:
- unittest.main(verbosity=2)
+ testRunner = None
+ unittest.main(testRunner=testRunner, verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/sql/tests/test_session.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests/test_session.py b/python/pyspark/sql/tests/test_session.py
index b811047..c6b9e0b 100644
--- a/python/pyspark/sql/tests/test_session.py
+++ b/python/pyspark/sql/tests/test_session.py
@@ -21,7 +21,7 @@ import unittest
from pyspark import SparkConf, SparkContext
from pyspark.sql import SparkSession, SQLContext, Row
from pyspark.testing.sqlutils import ReusedSQLTestCase
-from pyspark.tests import PySparkTestCase
+from pyspark.testing.utils import PySparkTestCase
class SparkSessionTests(ReusedSQLTestCase):
@@ -315,6 +315,7 @@ if __name__ == "__main__":
try:
import xmlrunner
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ testRunner = xmlrunner.XMLTestRunner(output='target/test-reports')
except ImportError:
- unittest.main(verbosity=2)
+ testRunner = None
+ unittest.main(testRunner=testRunner, verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/sql/tests/test_streaming.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests/test_streaming.py b/python/pyspark/sql/tests/test_streaming.py
index cc0cab4..4b71759 100644
--- a/python/pyspark/sql/tests/test_streaming.py
+++ b/python/pyspark/sql/tests/test_streaming.py
@@ -561,6 +561,7 @@ if __name__ == "__main__":
try:
import xmlrunner
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ testRunner = xmlrunner.XMLTestRunner(output='target/test-reports')
except ImportError:
- unittest.main(verbosity=2)
+ testRunner = None
+ unittest.main(testRunner=testRunner, verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/sql/tests/test_types.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests/test_types.py b/python/pyspark/sql/tests/test_types.py
index 3b32c58..fb673f2 100644
--- a/python/pyspark/sql/tests/test_types.py
+++ b/python/pyspark/sql/tests/test_types.py
@@ -939,6 +939,7 @@ if __name__ == "__main__":
try:
import xmlrunner
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ testRunner = xmlrunner.XMLTestRunner(output='target/test-reports')
except ImportError:
- unittest.main(verbosity=2)
+ testRunner = None
+ unittest.main(testRunner=testRunner, verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/sql/tests/test_udf.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests/test_udf.py b/python/pyspark/sql/tests/test_udf.py
index 630b215..d2dfb52 100644
--- a/python/pyspark/sql/tests/test_udf.py
+++ b/python/pyspark/sql/tests/test_udf.py
@@ -27,7 +27,7 @@ from pyspark.sql.functions import UserDefinedFunction
from pyspark.sql.types import *
from pyspark.sql.utils import AnalysisException
from pyspark.testing.sqlutils import ReusedSQLTestCase, test_compiled, test_not_compiled_message
-from pyspark.tests import QuietTest
+from pyspark.testing.utils import QuietTest
class UDFTests(ReusedSQLTestCase):
@@ -649,6 +649,7 @@ if __name__ == "__main__":
try:
import xmlrunner
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ testRunner = xmlrunner.XMLTestRunner(output='target/test-reports')
except ImportError:
- unittest.main(verbosity=2)
+ testRunner = None
+ unittest.main(testRunner=testRunner, verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/sql/tests/test_utils.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests/test_utils.py b/python/pyspark/sql/tests/test_utils.py
index 63a8614..5bb921d 100644
--- a/python/pyspark/sql/tests/test_utils.py
+++ b/python/pyspark/sql/tests/test_utils.py
@@ -49,6 +49,7 @@ if __name__ == "__main__":
try:
import xmlrunner
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
+ testRunner = xmlrunner.XMLTestRunner(output='target/test-reports')
except ImportError:
- unittest.main(verbosity=2)
+ testRunner = None
+ unittest.main(testRunner=testRunner, verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/test_broadcast.py
----------------------------------------------------------------------
diff --git a/python/pyspark/test_broadcast.py b/python/pyspark/test_broadcast.py
deleted file mode 100644
index a00329c..0000000
--- a/python/pyspark/test_broadcast.py
+++ /dev/null
@@ -1,126 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import os
-import random
-import tempfile
-import unittest
-
-try:
- import xmlrunner
-except ImportError:
- xmlrunner = None
-
-from pyspark.broadcast import Broadcast
-from pyspark.conf import SparkConf
-from pyspark.context import SparkContext
-from pyspark.java_gateway import launch_gateway
-from pyspark.serializers import ChunkedStream
-
-
-class BroadcastTest(unittest.TestCase):
-
- def tearDown(self):
- if getattr(self, "sc", None) is not None:
- self.sc.stop()
- self.sc = None
-
- def _test_encryption_helper(self, vs):
- """
- Creates a broadcast variables for each value in vs, and runs a simple job to make sure the
- value is the same when it's read in the executors. Also makes sure there are no task
- failures.
- """
- bs = [self.sc.broadcast(value=v) for v in vs]
- exec_values = self.sc.parallelize(range(2)).map(lambda x: [b.value for b in bs]).collect()
- for ev in exec_values:
- self.assertEqual(ev, vs)
- # make sure there are no task failures
- status = self.sc.statusTracker()
- for jid in status.getJobIdsForGroup():
- for sid in status.getJobInfo(jid).stageIds:
- stage_info = status.getStageInfo(sid)
- self.assertEqual(0, stage_info.numFailedTasks)
-
- def _test_multiple_broadcasts(self, *extra_confs):
- """
- Test broadcast variables make it OK to the executors. Tests multiple broadcast variables,
- and also multiple jobs.
- """
- conf = SparkConf()
- for key, value in extra_confs:
- conf.set(key, value)
- conf.setMaster("local-cluster[2,1,1024]")
- self.sc = SparkContext(conf=conf)
- self._test_encryption_helper([5])
- self._test_encryption_helper([5, 10, 20])
-
- def test_broadcast_with_encryption(self):
- self._test_multiple_broadcasts(("spark.io.encryption.enabled", "true"))
-
- def test_broadcast_no_encryption(self):
- self._test_multiple_broadcasts()
-
-
-class BroadcastFrameProtocolTest(unittest.TestCase):
-
- @classmethod
- def setUpClass(cls):
- gateway = launch_gateway(SparkConf())
- cls._jvm = gateway.jvm
- cls.longMessage = True
- random.seed(42)
-
- def _test_chunked_stream(self, data, py_buf_size):
- # write data using the chunked protocol from python.
- chunked_file = tempfile.NamedTemporaryFile(delete=False)
- dechunked_file = tempfile.NamedTemporaryFile(delete=False)
- dechunked_file.close()
- try:
- out = ChunkedStream(chunked_file, py_buf_size)
- out.write(data)
- out.close()
- # now try to read it in java
- jin = self._jvm.java.io.FileInputStream(chunked_file.name)
- jout = self._jvm.java.io.FileOutputStream(dechunked_file.name)
- self._jvm.DechunkedInputStream.dechunkAndCopyToOutput(jin, jout)
- # java should have decoded it back to the original data
- self.assertEqual(len(data), os.stat(dechunked_file.name).st_size)
- with open(dechunked_file.name, "rb") as f:
- byte = f.read(1)
- idx = 0
- while byte:
- self.assertEqual(data[idx], bytearray(byte)[0], msg="idx = " + str(idx))
- byte = f.read(1)
- idx += 1
- finally:
- os.unlink(chunked_file.name)
- os.unlink(dechunked_file.name)
-
- def test_chunked_stream(self):
- def random_bytes(n):
- return bytearray(random.getrandbits(8) for _ in range(n))
- for data_length in [1, 10, 100, 10000]:
- for buffer_length in [1, 2, 5, 8192]:
- self._test_chunked_stream(random_bytes(data_length), buffer_length)
-
-if __name__ == '__main__':
- from pyspark.test_broadcast import *
- if xmlrunner:
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
- else:
- unittest.main(verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/test_serializers.py
----------------------------------------------------------------------
diff --git a/python/pyspark/test_serializers.py b/python/pyspark/test_serializers.py
deleted file mode 100644
index 5b43729..0000000
--- a/python/pyspark/test_serializers.py
+++ /dev/null
@@ -1,90 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import io
-import math
-import struct
-import sys
-import unittest
-
-try:
- import xmlrunner
-except ImportError:
- xmlrunner = None
-
-from pyspark import serializers
-
-
-def read_int(b):
- return struct.unpack("!i", b)[0]
-
-
-def write_int(i):
- return struct.pack("!i", i)
-
-
-class SerializersTest(unittest.TestCase):
-
- def test_chunked_stream(self):
- original_bytes = bytearray(range(100))
- for data_length in [1, 10, 100]:
- for buffer_length in [1, 2, 3, 5, 20, 99, 100, 101, 500]:
- dest = ByteArrayOutput()
- stream_out = serializers.ChunkedStream(dest, buffer_length)
- stream_out.write(original_bytes[:data_length])
- stream_out.close()
- num_chunks = int(math.ceil(float(data_length) / buffer_length))
- # length for each chunk, and a final -1 at the very end
- exp_size = (num_chunks + 1) * 4 + data_length
- self.assertEqual(len(dest.buffer), exp_size)
- dest_pos = 0
- data_pos = 0
- for chunk_idx in range(num_chunks):
- chunk_length = read_int(dest.buffer[dest_pos:(dest_pos + 4)])
- if chunk_idx == num_chunks - 1:
- exp_length = data_length % buffer_length
- if exp_length == 0:
- exp_length = buffer_length
- else:
- exp_length = buffer_length
- self.assertEqual(chunk_length, exp_length)
- dest_pos += 4
- dest_chunk = dest.buffer[dest_pos:dest_pos + chunk_length]
- orig_chunk = original_bytes[data_pos:data_pos + chunk_length]
- self.assertEqual(dest_chunk, orig_chunk)
- dest_pos += chunk_length
- data_pos += chunk_length
- # ends with a -1
- self.assertEqual(dest.buffer[-4:], write_int(-1))
-
-
-class ByteArrayOutput(object):
- def __init__(self):
- self.buffer = bytearray()
-
- def write(self, b):
- self.buffer += b
-
- def close(self):
- pass
-
-if __name__ == '__main__':
- from pyspark.test_serializers import *
- if xmlrunner:
- unittest.main(testRunner=xmlrunner.XMLTestRunner(output='target/test-reports'), verbosity=2)
- else:
- unittest.main(verbosity=2)
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/testing/sqlutils.py
----------------------------------------------------------------------
diff --git a/python/pyspark/testing/sqlutils.py b/python/pyspark/testing/sqlutils.py
index 3951776..afc40cc 100644
--- a/python/pyspark/testing/sqlutils.py
+++ b/python/pyspark/testing/sqlutils.py
@@ -23,7 +23,7 @@ from contextlib import contextmanager
from pyspark.sql import SparkSession
from pyspark.sql.types import ArrayType, DoubleType, UserDefinedType, Row
-from pyspark.tests import ReusedPySparkTestCase
+from pyspark.testing.utils import ReusedPySparkTestCase
from pyspark.util import _exception_message
http://git-wip-us.apache.org/repos/asf/spark/blob/03306a6d/python/pyspark/testing/utils.py
----------------------------------------------------------------------
diff --git a/python/pyspark/testing/utils.py b/python/pyspark/testing/utils.py
new file mode 100644
index 0000000..7df0aca
--- /dev/null
+++ b/python/pyspark/testing/utils.py
@@ -0,0 +1,102 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import os
+import struct
+import sys
+import unittest
+
+from pyspark import SparkContext, SparkConf
+
+
+have_scipy = False
+have_numpy = False
+try:
+ import scipy.sparse
+ have_scipy = True
+except:
+ # No SciPy, but that's okay, we'll skip those tests
+ pass
+try:
+ import numpy as np
+ have_numpy = True
+except:
+ # No NumPy, but that's okay, we'll skip those tests
+ pass
+
+
+SPARK_HOME = os.environ["SPARK_HOME"]
+
+
+def read_int(b):
+ return struct.unpack("!i", b)[0]
+
+
+def write_int(i):
+ return struct.pack("!i", i)
+
+
+class QuietTest(object):
+ def __init__(self, sc):
+ self.log4j = sc._jvm.org.apache.log4j
+
+ def __enter__(self):
+ self.old_level = self.log4j.LogManager.getRootLogger().getLevel()
+ self.log4j.LogManager.getRootLogger().setLevel(self.log4j.Level.FATAL)
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.log4j.LogManager.getRootLogger().setLevel(self.old_level)
+
+
+class PySparkTestCase(unittest.TestCase):
+
+ def setUp(self):
+ self._old_sys_path = list(sys.path)
+ class_name = self.__class__.__name__
+ self.sc = SparkContext('local[4]', class_name)
+
+ def tearDown(self):
+ self.sc.stop()
+ sys.path = self._old_sys_path
+
+
+class ReusedPySparkTestCase(unittest.TestCase):
+
+ @classmethod
+ def conf(cls):
+ """
+ Override this in subclasses to supply a more specific conf
+ """
+ return SparkConf()
+
+ @classmethod
+ def setUpClass(cls):
+ cls.sc = SparkContext('local[4]', cls.__name__, conf=cls.conf())
+
+ @classmethod
+ def tearDownClass(cls):
+ cls.sc.stop()
+
+
+class ByteArrayOutput(object):
+ def __init__(self):
+ self.buffer = bytearray()
+
+ def write(self, b):
+ self.buffer += b
+
+ def close(self):
+ pass
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org