You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2016/04/14 20:03:32 UTC
spark git commit: [SPARK-14499][SQL][TEST] Drop Partition Does Not
Delete Data of External Tables
Repository: spark
Updated Branches:
refs/heads/master 1d04c86fc -> c971aee40
[SPARK-14499][SQL][TEST] Drop Partition Does Not Delete Data of External Tables
#### What changes were proposed in this pull request?
This PR is to add a test to ensure drop partitions of an external table will not delete data.
cc yhuai andrewor14
#### How was this patch tested?
N/A
Author: gatorsmile <ga...@gmail.com>
This patch had conflicts when merged, resolved by
Committer: Andrew Or <an...@databricks.com>
Closes #12350 from gatorsmile/testDropPartition.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c971aee4
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c971aee4
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c971aee4
Branch: refs/heads/master
Commit: c971aee40d806ed02d3d6a5cc478b63654052e54
Parents: 1d04c86
Author: gatorsmile <ga...@gmail.com>
Authored: Thu Apr 14 11:03:19 2016 -0700
Committer: Andrew Or <an...@databricks.com>
Committed: Thu Apr 14 11:03:19 2016 -0700
----------------------------------------------------------------------
.../spark/sql/hive/execution/HiveDDLSuite.scala | 67 ++++++++++++++++++++
1 file changed, 67 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/c971aee4/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index 249dcdf..206d911 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql.hive.execution
+import java.io.File
+
import org.apache.hadoop.fs.Path
import org.apache.spark.sql.{AnalysisException, QueryTest, SaveMode}
@@ -126,6 +128,71 @@ class HiveDDLSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
}
}
+ test("add/drop partitions - external table") {
+ val catalog = hiveContext.sessionState.catalog
+ withTempDir { tmpDir =>
+ val basePath = tmpDir.getCanonicalPath
+ val partitionPath_1stCol_part1 = new File(basePath + "/ds=2008-04-08")
+ val partitionPath_1stCol_part2 = new File(basePath + "/ds=2008-04-09")
+ val partitionPath_part1 = new File(basePath + "/ds=2008-04-08/hr=11")
+ val partitionPath_part2 = new File(basePath + "/ds=2008-04-09/hr=11")
+ val partitionPath_part3 = new File(basePath + "/ds=2008-04-08/hr=12")
+ val partitionPath_part4 = new File(basePath + "/ds=2008-04-09/hr=12")
+ val dirSet =
+ tmpDir :: partitionPath_1stCol_part1 :: partitionPath_1stCol_part2 ::
+ partitionPath_part1 :: partitionPath_part2 :: partitionPath_part3 ::
+ partitionPath_part4 :: Nil
+
+ val externalTab = "extTable_with_partitions"
+ withTable(externalTab) {
+ assert(tmpDir.listFiles.isEmpty)
+ sql(
+ s"""
+ |CREATE EXTERNAL TABLE $externalTab (key INT, value STRING)
+ |PARTITIONED BY (ds STRING, hr STRING)
+ |LOCATION '$basePath'
+ """.stripMargin)
+
+ // Before data insertion, all the directory are empty
+ assert(dirSet.forall(dir => dir.listFiles == null || dir.listFiles.isEmpty))
+
+ for (ds <- Seq("2008-04-08", "2008-04-09"); hr <- Seq("11", "12")) {
+ sql(
+ s"""
+ |INSERT OVERWRITE TABLE $externalTab
+ |partition (ds='$ds',hr='$hr')
+ |SELECT 1, 'a'
+ """.stripMargin)
+ }
+
+ val hiveTable = catalog.getTableMetadata(TableIdentifier(externalTab, Some("default")))
+ assert(hiveTable.tableType == CatalogTableType.EXTERNAL_TABLE)
+ // After data insertion, all the directory are not empty
+ assert(dirSet.forall(dir => dir.listFiles.nonEmpty))
+
+ sql(
+ s"""
+ |ALTER TABLE $externalTab DROP PARTITION (ds='2008-04-08'),
+ |PARTITION (ds='2008-04-09', hr='12')
+ """.stripMargin)
+ assert(catalog.listPartitions(TableIdentifier(externalTab)).map(_.spec).toSet ==
+ Set(Map("ds" -> "2008-04-09", "hr" -> "11")))
+ // drop partition will not delete the data of external table
+ assert(dirSet.forall(dir => dir.listFiles.nonEmpty))
+
+ sql(s"ALTER TABLE $externalTab ADD PARTITION (ds='2008-04-08', hr='12')")
+ assert(catalog.listPartitions(TableIdentifier(externalTab)).map(_.spec).toSet ==
+ Set(Map("ds" -> "2008-04-08", "hr" -> "12"), Map("ds" -> "2008-04-09", "hr" -> "11")))
+ // add partition will not delete the data
+ assert(dirSet.forall(dir => dir.listFiles.nonEmpty))
+
+ sql(s"DROP TABLE $externalTab")
+ // drop table will not delete the data of external table
+ assert(dirSet.forall(dir => dir.listFiles.nonEmpty))
+ }
+ }
+ }
+
test("drop views") {
withTable("tab1") {
val tabName = "tab1"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org