You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2016/10/11 06:30:01 UTC
spark git commit: [SPARK-17808][PYSPARK] Upgraded version of Pyrolite
to 4.13
Repository: spark
Updated Branches:
refs/heads/master 19401a203 -> 658c7147f
[SPARK-17808][PYSPARK] Upgraded version of Pyrolite to 4.13
## What changes were proposed in this pull request?
Upgraded to a newer version of Pyrolite which supports serialization of a BinaryType StructField for PySpark.SQL
## How was this patch tested?
Added a unit test which fails with a raised ValueError when using the previous version of Pyrolite 4.9 and Python3
Author: Bryan Cutler <cu...@gmail.com>
Closes #15386 from BryanCutler/pyrolite-upgrade-SPARK-17808.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/658c7147
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/658c7147
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/658c7147
Branch: refs/heads/master
Commit: 658c7147f5bf637f36e8c66b9207d94b1e7c74c5
Parents: 19401a2
Author: Bryan Cutler <cu...@gmail.com>
Authored: Tue Oct 11 08:29:52 2016 +0200
Committer: Sean Owen <so...@cloudera.com>
Committed: Tue Oct 11 08:29:52 2016 +0200
----------------------------------------------------------------------
core/pom.xml | 2 +-
dev/deps/spark-deps-hadoop-2.2 | 2 +-
dev/deps/spark-deps-hadoop-2.3 | 2 +-
dev/deps/spark-deps-hadoop-2.4 | 2 +-
dev/deps/spark-deps-hadoop-2.6 | 2 +-
dev/deps/spark-deps-hadoop-2.7 | 2 +-
python/pyspark/sql/tests.py | 8 ++++++++
7 files changed, 14 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/658c7147/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 9a4f234..205bbc5 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -320,7 +320,7 @@
<dependency>
<groupId>net.razorvine</groupId>
<artifactId>pyrolite</artifactId>
- <version>4.9</version>
+ <version>4.13</version>
<exclusions>
<exclusion>
<groupId>net.razorvine</groupId>
http://git-wip-us.apache.org/repos/asf/spark/blob/658c7147/dev/deps/spark-deps-hadoop-2.2
----------------------------------------------------------------------
diff --git a/dev/deps/spark-deps-hadoop-2.2 b/dev/deps/spark-deps-hadoop-2.2
index f4f92c6..b30f8c3 100644
--- a/dev/deps/spark-deps-hadoop-2.2
+++ b/dev/deps/spark-deps-hadoop-2.2
@@ -141,7 +141,7 @@ pmml-model-1.2.15.jar
pmml-schema-1.2.15.jar
protobuf-java-2.5.0.jar
py4j-0.10.3.jar
-pyrolite-4.9.jar
+pyrolite-4.13.jar
scala-compiler-2.11.8.jar
scala-library-2.11.8.jar
scala-parser-combinators_2.11-1.0.4.jar
http://git-wip-us.apache.org/repos/asf/spark/blob/658c7147/dev/deps/spark-deps-hadoop-2.3
----------------------------------------------------------------------
diff --git a/dev/deps/spark-deps-hadoop-2.3 b/dev/deps/spark-deps-hadoop-2.3
index 3db013f..5b3a765 100644
--- a/dev/deps/spark-deps-hadoop-2.3
+++ b/dev/deps/spark-deps-hadoop-2.3
@@ -148,7 +148,7 @@ pmml-model-1.2.15.jar
pmml-schema-1.2.15.jar
protobuf-java-2.5.0.jar
py4j-0.10.3.jar
-pyrolite-4.9.jar
+pyrolite-4.13.jar
scala-compiler-2.11.8.jar
scala-library-2.11.8.jar
scala-parser-combinators_2.11-1.0.4.jar
http://git-wip-us.apache.org/repos/asf/spark/blob/658c7147/dev/deps/spark-deps-hadoop-2.4
----------------------------------------------------------------------
diff --git a/dev/deps/spark-deps-hadoop-2.4 b/dev/deps/spark-deps-hadoop-2.4
index 7171010..e323efe 100644
--- a/dev/deps/spark-deps-hadoop-2.4
+++ b/dev/deps/spark-deps-hadoop-2.4
@@ -148,7 +148,7 @@ pmml-model-1.2.15.jar
pmml-schema-1.2.15.jar
protobuf-java-2.5.0.jar
py4j-0.10.3.jar
-pyrolite-4.9.jar
+pyrolite-4.13.jar
scala-compiler-2.11.8.jar
scala-library-2.11.8.jar
scala-parser-combinators_2.11-1.0.4.jar
http://git-wip-us.apache.org/repos/asf/spark/blob/658c7147/dev/deps/spark-deps-hadoop-2.6
----------------------------------------------------------------------
diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6
index cb30fda..77d97e5 100644
--- a/dev/deps/spark-deps-hadoop-2.6
+++ b/dev/deps/spark-deps-hadoop-2.6
@@ -156,7 +156,7 @@ pmml-model-1.2.15.jar
pmml-schema-1.2.15.jar
protobuf-java-2.5.0.jar
py4j-0.10.3.jar
-pyrolite-4.9.jar
+pyrolite-4.13.jar
scala-compiler-2.11.8.jar
scala-library-2.11.8.jar
scala-parser-combinators_2.11-1.0.4.jar
http://git-wip-us.apache.org/repos/asf/spark/blob/658c7147/dev/deps/spark-deps-hadoop-2.7
----------------------------------------------------------------------
diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7
index 9008aa8..572edfa 100644
--- a/dev/deps/spark-deps-hadoop-2.7
+++ b/dev/deps/spark-deps-hadoop-2.7
@@ -157,7 +157,7 @@ pmml-model-1.2.15.jar
pmml-schema-1.2.15.jar
protobuf-java-2.5.0.jar
py4j-0.10.3.jar
-pyrolite-4.9.jar
+pyrolite-4.13.jar
scala-compiler-2.11.8.jar
scala-library-2.11.8.jar
scala-parser-combinators_2.11-1.0.4.jar
http://git-wip-us.apache.org/repos/asf/spark/blob/658c7147/python/pyspark/sql/tests.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py
index 7b6f9f0..86c590d 100644
--- a/python/pyspark/sql/tests.py
+++ b/python/pyspark/sql/tests.py
@@ -1708,6 +1708,14 @@ class SQLTests(ReusedPySparkTestCase):
count = df.count()
self.assertEquals(count, 4)
+ def test_BinaryType_serialization(self):
+ # Pyrolite version <= 4.9 could not serialize BinaryType with Python3 SPARK-17808
+ schema = StructType([StructField('mybytes', BinaryType())])
+ data = [[bytearray(b'here is my data')],
+ [bytearray(b'and here is some more')]]
+ df = self.spark.createDataFrame(data, schema=schema)
+ df.collect()
+
class HiveSparkSubmitTests(SparkSubmitTests):
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org