You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2015/07/05 07:52:56 UTC
spark git commit: [MINOR] [SQL] Minor fix for CatalystSchemaConverter
Repository: spark
Updated Branches:
refs/heads/master c991ef5ab -> 2b820f2a4
[MINOR] [SQL] Minor fix for CatalystSchemaConverter
ping liancheng
Author: Liang-Chi Hsieh <vi...@gmail.com>
Closes #7224 from viirya/few_fix_catalystschema and squashes the following commits:
d994330 [Liang-Chi Hsieh] Minor fix for CatalystSchemaConverter.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2b820f2a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2b820f2a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2b820f2a
Branch: refs/heads/master
Commit: 2b820f2a4bf9b154762e7516a5b0485322799da9
Parents: c991ef5
Author: Liang-Chi Hsieh <vi...@gmail.com>
Authored: Sat Jul 4 22:52:50 2015 -0700
Committer: Cheng Lian <li...@databricks.com>
Committed: Sat Jul 4 22:52:50 2015 -0700
----------------------------------------------------------------------
.../src/main/scala/org/apache/spark/sql/SQLConf.scala | 2 +-
.../spark/sql/parquet/CatalystSchemaConverter.scala | 12 ++++++------
2 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/2b820f2a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala
index 2c258b6..6005d35 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLConf.scala
@@ -273,7 +273,7 @@ private[spark] object SQLConf {
val PARQUET_FOLLOW_PARQUET_FORMAT_SPEC = booleanConf(
key = "spark.sql.parquet.followParquetFormatSpec",
defaultValue = Some(false),
- doc = "Wether to stick to Parquet format specification when converting Parquet schema to " +
+ doc = "Whether to stick to Parquet format specification when converting Parquet schema to " +
"Spark SQL schema and vice versa. Sticks to the specification if set to true; falls back " +
"to compatible mode if set to false.",
isPublic = false)
http://git-wip-us.apache.org/repos/asf/spark/blob/2b820f2a/sql/core/src/main/scala/org/apache/spark/sql/parquet/CatalystSchemaConverter.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/parquet/CatalystSchemaConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/parquet/CatalystSchemaConverter.scala
index 2be7c64..4ab274e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/parquet/CatalystSchemaConverter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/parquet/CatalystSchemaConverter.scala
@@ -142,7 +142,7 @@ private[parquet] class CatalystSchemaConverter(
DecimalType(precision, scale)
}
- field.getPrimitiveTypeName match {
+ typeName match {
case BOOLEAN => BooleanType
case FLOAT => FloatType
@@ -150,7 +150,7 @@ private[parquet] class CatalystSchemaConverter(
case DOUBLE => DoubleType
case INT32 =>
- field.getOriginalType match {
+ originalType match {
case INT_8 => ByteType
case INT_16 => ShortType
case INT_32 | null => IntegerType
@@ -161,7 +161,7 @@ private[parquet] class CatalystSchemaConverter(
}
case INT64 =>
- field.getOriginalType match {
+ originalType match {
case INT_64 | null => LongType
case DECIMAL => makeDecimalType(maxPrecisionForBytes(8))
case TIMESTAMP_MILLIS => typeNotImplemented()
@@ -176,7 +176,7 @@ private[parquet] class CatalystSchemaConverter(
TimestampType
case BINARY =>
- field.getOriginalType match {
+ originalType match {
case UTF8 | ENUM => StringType
case null if assumeBinaryIsString => StringType
case null => BinaryType
@@ -185,7 +185,7 @@ private[parquet] class CatalystSchemaConverter(
}
case FIXED_LEN_BYTE_ARRAY =>
- field.getOriginalType match {
+ originalType match {
case DECIMAL => makeDecimalType(maxPrecisionForBytes(field.getTypeLength))
case INTERVAL => typeNotImplemented()
case _ => illegalType()
@@ -261,7 +261,7 @@ private[parquet] class CatalystSchemaConverter(
// Here we implement Parquet LIST backwards-compatibility rules.
// See: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#backward-compatibility-rules
// scalastyle:on
- private def isElementType(repeatedType: Type, parentName: String) = {
+ private def isElementType(repeatedType: Type, parentName: String): Boolean = {
{
// For legacy 2-level list types with primitive element type, e.g.:
//
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org