You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2015/06/30 03:50:17 UTC
spark git commit: [SPARK-8669] [SQL] Fix crash with BINARY (ENUM)
fields with Parquet 1.7
Repository: spark
Updated Branches:
refs/heads/master ecacb1e88 -> 4915e9e3b
[SPARK-8669] [SQL] Fix crash with BINARY (ENUM) fields with Parquet 1.7
Patch to fix crash with BINARY fields with ENUM original types.
Author: Steven She <st...@canopylabs.com>
Closes #7048 from stevencanopy/SPARK-8669 and squashes the following commits:
2e72979 [Steven She] [SPARK-8669] [SQL] Fix crash with BINARY (ENUM) fields with Parquet 1.7
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4915e9e3
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4915e9e3
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4915e9e3
Branch: refs/heads/master
Commit: 4915e9e3bffb57eac319ef2173b4a6ae4073d25e
Parents: ecacb1e
Author: Steven She <st...@canopylabs.com>
Authored: Mon Jun 29 18:50:09 2015 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Mon Jun 29 18:50:09 2015 -0700
----------------------------------------------------------------------
.../apache/spark/sql/parquet/CatalystSchemaConverter.scala | 2 +-
.../org/apache/spark/sql/parquet/ParquetSchemaSuite.scala | 8 ++++++++
2 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/4915e9e3/sql/core/src/main/scala/org/apache/spark/sql/parquet/CatalystSchemaConverter.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/parquet/CatalystSchemaConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/parquet/CatalystSchemaConverter.scala
index 4fd3e93..2be7c64 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/parquet/CatalystSchemaConverter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/parquet/CatalystSchemaConverter.scala
@@ -177,7 +177,7 @@ private[parquet] class CatalystSchemaConverter(
case BINARY =>
field.getOriginalType match {
- case UTF8 => StringType
+ case UTF8 | ENUM => StringType
case null if assumeBinaryIsString => StringType
case null => BinaryType
case DECIMAL => makeDecimalType()
http://git-wip-us.apache.org/repos/asf/spark/blob/4915e9e3/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala
index d0bfcde..35d3c33 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/parquet/ParquetSchemaSuite.scala
@@ -161,6 +161,14 @@ class ParquetSchemaInferenceSuite extends ParquetSchemaTest {
""".stripMargin,
binaryAsString = true)
+ testSchemaInference[Tuple1[String]](
+ "binary enum as string",
+ """
+ |message root {
+ | optional binary _1 (ENUM);
+ |}
+ """.stripMargin)
+
testSchemaInference[Tuple1[Seq[Int]]](
"non-nullable array - non-standard",
"""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org