You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/07/31 22:05:31 UTC
git commit: [SPARK-2523] [SQL] Hadoop table scan bug fixing (fix
failing Jenkins maven test)
Repository: spark
Updated Branches:
refs/heads/master dc0865bc7 -> 49b361298
[SPARK-2523] [SQL] Hadoop table scan bug fixing (fix failing Jenkins maven test)
This PR tries to resolve the broken Jenkins maven test issue introduced by #1439. Now, we create a single query test to run both the setup work and the test query.
Author: Yin Huai <hu...@cse.ohio-state.edu>
Closes #1669 from yhuai/SPARK-2523-fixTest and squashes the following commits:
358af1a [Yin Huai] Make partition_based_table_scan_with_different_serde run atomically.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/49b36129
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/49b36129
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/49b36129
Branch: refs/heads/master
Commit: 49b361298b09d415de1857846367913495aecfa6
Parents: dc0865b
Author: Yin Huai <hu...@cse.ohio-state.edu>
Authored: Thu Jul 31 13:05:24 2014 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Thu Jul 31 13:05:24 2014 -0700
----------------------------------------------------------------------
...ent_serde-0-1436cccda63b78dd6e43a399da6cc474 | 0
...ent_serde-0-8caed2a6e80250a6d38a59388679c298 | 2 -
...ent_serde-1-8d9bf54373f45bc35f8cb6e82771b154 | 0
...ent_serde-2-7816c17905012cf381abf93d230faa8d | 0
...ent_serde-3-90089a6db3c3d8ee5ff5ea6b9153b3cc | 0
...ent_serde-4-8caed2a6e80250a6d38a59388679c298 | 2 +
.../sql/hive/execution/HiveTableScanSuite.scala | 45 +++++++++-----------
7 files changed, 21 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/49b36129/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-0-1436cccda63b78dd6e43a399da6cc474
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-0-1436cccda63b78dd6e43a399da6cc474 b/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-0-1436cccda63b78dd6e43a399da6cc474
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/spark/blob/49b36129/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-0-8caed2a6e80250a6d38a59388679c298
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-0-8caed2a6e80250a6d38a59388679c298 b/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-0-8caed2a6e80250a6d38a59388679c298
deleted file mode 100644
index f369f21..0000000
--- a/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-0-8caed2a6e80250a6d38a59388679c298
+++ /dev/null
@@ -1,2 +0,0 @@
-100 100 2010-01-01
-200 200 2010-01-02
http://git-wip-us.apache.org/repos/asf/spark/blob/49b36129/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-1-8d9bf54373f45bc35f8cb6e82771b154
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-1-8d9bf54373f45bc35f8cb6e82771b154 b/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-1-8d9bf54373f45bc35f8cb6e82771b154
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/spark/blob/49b36129/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-2-7816c17905012cf381abf93d230faa8d
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-2-7816c17905012cf381abf93d230faa8d b/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-2-7816c17905012cf381abf93d230faa8d
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/spark/blob/49b36129/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-3-90089a6db3c3d8ee5ff5ea6b9153b3cc
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-3-90089a6db3c3d8ee5ff5ea6b9153b3cc b/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-3-90089a6db3c3d8ee5ff5ea6b9153b3cc
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/spark/blob/49b36129/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-4-8caed2a6e80250a6d38a59388679c298
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-4-8caed2a6e80250a6d38a59388679c298 b/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-4-8caed2a6e80250a6d38a59388679c298
new file mode 100644
index 0000000..f369f21
--- /dev/null
+++ b/sql/hive/src/test/resources/golden/partition_based_table_scan_with_different_serde-4-8caed2a6e80250a6d38a59388679c298
@@ -0,0 +1,2 @@
+100 100 2010-01-01
+200 200 2010-01-02
http://git-wip-us.apache.org/repos/asf/spark/blob/49b36129/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala
index bcb00f8..c573672 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala
@@ -17,32 +17,25 @@
package org.apache.spark.sql.hive.execution
-import org.scalatest.{BeforeAndAfterAll, FunSuite}
-
-import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.sql.hive.test.TestHive
-
class HiveTableScanSuite extends HiveComparisonTest {
- // MINOR HACK: You must run a query before calling reset the first time.
- TestHive.hql("SHOW TABLES")
- TestHive.reset()
-
- TestHive.hql("""CREATE TABLE part_scan_test (key STRING, value STRING) PARTITIONED BY (ds STRING)
- | ROW FORMAT SERDE
- | 'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe'
- | STORED AS RCFILE
- """.stripMargin)
- TestHive.hql("""FROM src
- | INSERT INTO TABLE part_scan_test PARTITION (ds='2010-01-01')
- | SELECT 100,100 LIMIT 1
- """.stripMargin)
- TestHive.hql("""ALTER TABLE part_scan_test SET SERDE
- | 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe'
- """.stripMargin)
- TestHive.hql("""FROM src INSERT INTO TABLE part_scan_test PARTITION (ds='2010-01-02')
- | SELECT 200,200 LIMIT 1
- """.stripMargin)
- createQueryTest("partition_based_table_scan_with_different_serde",
- "SELECT * from part_scan_test", false)
+ createQueryTest("partition_based_table_scan_with_different_serde",
+ """
+ |CREATE TABLE part_scan_test (key STRING, value STRING) PARTITIONED BY (ds STRING)
+ |ROW FORMAT SERDE
+ |'org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe'
+ |STORED AS RCFILE;
+ |
+ |FROM src
+ |INSERT INTO TABLE part_scan_test PARTITION (ds='2010-01-01')
+ |SELECT 100,100 LIMIT 1;
+ |
+ |ALTER TABLE part_scan_test SET SERDE
+ |'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe';
+ |
+ |FROM src INSERT INTO TABLE part_scan_test PARTITION (ds='2010-01-02')
+ |SELECT 200,200 LIMIT 1;
+ |
+ |SELECT * from part_scan_test;
+ """.stripMargin)
}