You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2017/12/11 19:36:18 UTC
spark git commit: [SPARK-22642][SQL] the createdTempDir will not be
deleted if an exception occurs, should delete it with try-finally.
Repository: spark
Updated Branches:
refs/heads/master 6cc7021a4 -> bf20abb2d
[SPARK-22642][SQL] the createdTempDir will not be deleted if an exception occurs, should delete it with try-finally.
## What changes were proposed in this pull request?
We found staging directories will not be dropped sometimes in our production environment.
The createdTempDir will not be deleted if an exception occurs, we should delete createdTempDir with try-finally.
This PR is follow-up SPARK-18703.
## How was this patch tested?
exist tests
Author: zuotingbing <zu...@zte.com.cn>
Closes #19841 from zuotingbing/SPARK-stagedir.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bf20abb2
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bf20abb2
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bf20abb2
Branch: refs/heads/master
Commit: bf20abb2dc084b743289ea24fc8d6e47d6e6c0dd
Parents: 6cc7021
Author: zuotingbing <zu...@zte.com.cn>
Authored: Mon Dec 11 13:36:15 2017 -0600
Committer: Sean Owen <so...@cloudera.com>
Committed: Mon Dec 11 13:36:15 2017 -0600
----------------------------------------------------------------------
.../hive/execution/InsertIntoHiveTable.scala | 47 +++++++++++++-------
1 file changed, 30 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/bf20abb2/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
index 56e10bc..b46addb 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
@@ -17,13 +17,14 @@
package org.apache.spark.sql.hive.execution
+import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.hadoop.hive.ql.ErrorMsg
import org.apache.hadoop.hive.ql.plan.TableDesc
import org.apache.spark.SparkException
import org.apache.spark.sql.{AnalysisException, Dataset, Row, SparkSession}
-import org.apache.spark.sql.catalyst.catalog.CatalogTable
+import org.apache.spark.sql.catalyst.catalog.{CatalogTable, ExternalCatalog}
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.command.CommandUtils
@@ -91,6 +92,34 @@ case class InsertIntoHiveTable(
)
val tableLocation = hiveQlTable.getDataLocation
val tmpLocation = getExternalTmpPath(sparkSession, hadoopConf, tableLocation)
+
+ try {
+ processInsert(sparkSession, externalCatalog, hadoopConf, tableDesc, tmpLocation)
+ } finally {
+ // Attempt to delete the staging directory and the inclusive files. If failed, the files are
+ // expected to be dropped at the normal termination of VM since deleteOnExit is used.
+ deleteExternalTmpPath(hadoopConf)
+ }
+
+ // un-cache this table.
+ sparkSession.catalog.uncacheTable(table.identifier.quotedString)
+ sparkSession.sessionState.catalog.refreshTable(table.identifier)
+
+ CommandUtils.updateTableStats(sparkSession, table)
+
+ // It would be nice to just return the childRdd unchanged so insert operations could be chained,
+ // however for now we return an empty list to simplify compatibility checks with hive, which
+ // does not return anything for insert operations.
+ // TODO: implement hive compatibility as rules.
+ Seq.empty[Row]
+ }
+
+ private def processInsert(
+ sparkSession: SparkSession,
+ externalCatalog: ExternalCatalog,
+ hadoopConf: Configuration,
+ tableDesc: TableDesc,
+ tmpLocation: Path): Unit = {
val fileSinkConf = new FileSinkDesc(tmpLocation.toString, tableDesc, false)
val numDynamicPartitions = partition.values.count(_.isEmpty)
@@ -231,21 +260,5 @@ case class InsertIntoHiveTable(
overwrite,
isSrcLocal = false)
}
-
- // Attempt to delete the staging directory and the inclusive files. If failed, the files are
- // expected to be dropped at the normal termination of VM since deleteOnExit is used.
- deleteExternalTmpPath(hadoopConf)
-
- // un-cache this table.
- sparkSession.catalog.uncacheTable(table.identifier.quotedString)
- sparkSession.sessionState.catalog.refreshTable(table.identifier)
-
- CommandUtils.updateTableStats(sparkSession, table)
-
- // It would be nice to just return the childRdd unchanged so insert operations could be chained,
- // however for now we return an empty list to simplify compatibility checks with hive, which
- // does not return anything for insert operations.
- // TODO: implement hive compatibility as rules.
- Seq.empty[Row]
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org