You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2015/02/05 00:26:14 UTC
spark git commit: [SPARK-5118][SQL] Fix: create table test stored as
parquet as select ..
Repository: spark
Updated Branches:
refs/heads/master 548c9c2b2 -> e0490e271
[SPARK-5118][SQL] Fix: create table test stored as parquet as select ..
Author: guowei2 <gu...@asiainfo.com>
Closes #3921 from guowei2/SPARK-5118 and squashes the following commits:
b1ba3be [guowei2] add table file check in test case
9da56f8 [guowei2] test case only run in Shim13
112a0b6 [guowei2] add test case
187c7d8 [guowei2] Fix: create table test stored as parquet as select ..
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e0490e27
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e0490e27
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e0490e27
Branch: refs/heads/master
Commit: e0490e271d078aa55d7c7583e2ba80337ed1b0c4
Parents: 548c9c2
Author: guowei2 <gu...@asiainfo.com>
Authored: Wed Feb 4 15:26:10 2015 -0800
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Wed Feb 4 15:26:10 2015 -0800
----------------------------------------------------------------------
.../org/apache/spark/sql/hive/HiveQl.scala | 1 +
.../sql/hive/execution/SQLQuerySuite.scala | 29 +++++++++++++++++++-
2 files changed, 29 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/e0490e27/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
index 9b5c081..286d139 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
@@ -554,6 +554,7 @@ https://cwiki.apache.org/confluence/display/Hive/Enhanced+Aggregation%2C+Cube%2C
"TOK_TBLTEXTFILE", // Stored as TextFile
"TOK_TBLRCFILE", // Stored as RCFile
"TOK_TBLORCFILE", // Stored as ORC File
+ "TOK_TBLPARQUETFILE", // Stored as PARQUET
"TOK_TABLEFILEFORMAT", // User-provided InputFormat and OutputFormat
"TOK_STORAGEHANDLER", // Storage handler
"TOK_TABLELOCATION",
http://git-wip-us.apache.org/repos/asf/spark/blob/e0490e27/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index 4efe0c5..268e5f6 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -22,6 +22,8 @@ import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.Row
import org.apache.spark.sql.hive.test.TestHive._
import org.apache.spark.sql.types._
+import org.apache.spark.util.Utils
+import org.apache.spark.sql.hive.HiveShim
case class Nested1(f1: Nested2)
case class Nested2(f2: Nested3)
@@ -102,6 +104,31 @@ class SQLQuerySuite extends QueryTest {
"org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe",
"serde_p1=p1", "serde_p2=p2", "tbl_p1=p11", "tbl_p2=p22","MANAGED_TABLE"
)
+
+ if (HiveShim.version =="0.13.1") {
+ sql(
+ """CREATE TABLE ctas5
+ | STORED AS parquet AS
+ | SELECT key, value
+ | FROM src
+ | ORDER BY key, value""".stripMargin).collect
+
+ checkExistence(sql("DESC EXTENDED ctas5"), true,
+ "name:key", "type:string", "name:value", "ctas5",
+ "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
+ "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat",
+ "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe",
+ "MANAGED_TABLE"
+ )
+
+ val default = getConf("spark.sql.hive.convertMetastoreParquet", "true")
+ // use the Hive SerDe for parquet tables
+ sql("set spark.sql.hive.convertMetastoreParquet = false")
+ checkAnswer(
+ sql("SELECT key, value FROM ctas5 ORDER BY key, value"),
+ sql("SELECT key, value FROM src ORDER BY key, value").collect().toSeq)
+ sql(s"set spark.sql.hive.convertMetastoreParquet = $default")
+ }
}
test("command substitution") {
@@ -159,7 +186,7 @@ class SQLQuerySuite extends QueryTest {
test("test CTAS") {
checkAnswer(sql("CREATE TABLE test_ctas_123 AS SELECT key, value FROM src"), Seq.empty[Row])
checkAnswer(
- sql("SELECT key, value FROM test_ctas_123 ORDER BY key"),
+ sql("SELECT key, value FROM test_ctas_123 ORDER BY key"),
sql("SELECT key, value FROM src ORDER BY key").collect().toSeq)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org