You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2018/01/09 02:15:11 UTC
spark git commit: [SPARK-22972] Couldn't find corresponding Hive
SerDe for data source provider org.apache.spark.sql.hive.orc
Repository: spark
Updated Branches:
refs/heads/master 4f7e75883 -> 68ce792b5
[SPARK-22972] Couldn't find corresponding Hive SerDe for data source provider org.apache.spark.sql.hive.orc
## What changes were proposed in this pull request?
Fix the warning: Couldn't find corresponding Hive SerDe for data source provider org.apache.spark.sql.hive.orc.
## How was this patch tested?
test("SPARK-22972: hive orc source")
assert(HiveSerDe.sourceToSerDe("org.apache.spark.sql.hive.orc")
.equals(HiveSerDe.sourceToSerDe("orc")))
Author: xubo245 <60...@qq.com>
Closes #20165 from xubo245/HiveSerDe.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/68ce792b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/68ce792b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/68ce792b
Branch: refs/heads/master
Commit: 68ce792b5857f0291154f524ac651036db868bb9
Parents: 4f7e758
Author: xubo245 <60...@qq.com>
Authored: Tue Jan 9 10:15:01 2018 +0800
Committer: gatorsmile <ga...@gmail.com>
Committed: Tue Jan 9 10:15:01 2018 +0800
----------------------------------------------------------------------
.../apache/spark/sql/internal/HiveSerDe.scala | 1 +
.../spark/sql/hive/orc/HiveOrcSourceSuite.scala | 29 ++++++++++++++++++++
2 files changed, 30 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/68ce792b/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala
index b9515ec..dac4636 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/HiveSerDe.scala
@@ -73,6 +73,7 @@ object HiveSerDe {
val key = source.toLowerCase(Locale.ROOT) match {
case s if s.startsWith("org.apache.spark.sql.parquet") => "parquet"
case s if s.startsWith("org.apache.spark.sql.orc") => "orc"
+ case s if s.startsWith("org.apache.spark.sql.hive.orc") => "orc"
case s if s.equals("orcfile") => "orc"
case s if s.equals("parquetfile") => "parquet"
case s if s.equals("avrofile") => "avro"
http://git-wip-us.apache.org/repos/asf/spark/blob/68ce792b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
index 17b7d8c..d556a03 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/orc/HiveOrcSourceSuite.scala
@@ -20,8 +20,10 @@ package org.apache.spark.sql.hive.orc
import java.io.File
import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.execution.datasources.orc.OrcSuite
import org.apache.spark.sql.hive.test.TestHiveSingleton
+import org.apache.spark.sql.internal.HiveSerDe
import org.apache.spark.util.Utils
class HiveOrcSourceSuite extends OrcSuite with TestHiveSingleton {
@@ -62,6 +64,33 @@ class HiveOrcSourceSuite extends OrcSuite with TestHiveSingleton {
""".stripMargin)
}
+ test("SPARK-22972: hive orc source") {
+ val tableName = "normal_orc_as_source_hive"
+ withTable(tableName) {
+ sql(
+ s"""
+ |CREATE TABLE $tableName
+ |USING org.apache.spark.sql.hive.orc
+ |OPTIONS (
+ | PATH '${new File(orcTableAsDir.getAbsolutePath).toURI}'
+ |)
+ """.stripMargin)
+
+ val tableMetadata = spark.sessionState.catalog.getTableMetadata(
+ TableIdentifier(tableName))
+ assert(tableMetadata.storage.inputFormat ==
+ Option("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"))
+ assert(tableMetadata.storage.outputFormat ==
+ Option("org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat"))
+ assert(tableMetadata.storage.serde ==
+ Option("org.apache.hadoop.hive.ql.io.orc.OrcSerde"))
+ assert(HiveSerDe.sourceToSerDe("org.apache.spark.sql.hive.orc")
+ .equals(HiveSerDe.sourceToSerDe("orc")))
+ assert(HiveSerDe.sourceToSerDe("org.apache.spark.sql.orc")
+ .equals(HiveSerDe.sourceToSerDe("orc")))
+ }
+ }
+
test("SPARK-19459/SPARK-18220: read char/varchar column written by Hive") {
val location = Utils.createTempDir()
val uri = location.toURI
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org