You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2018/02/12 22:58:35 UTC
spark git commit: [SPARK-23378][SQL] move setCurrentDatabase from
HiveExternalCatalog to HiveClientImpl
Repository: spark
Updated Branches:
refs/heads/master 0c66fe4f2 -> fba01b9a6
[SPARK-23378][SQL] move setCurrentDatabase from HiveExternalCatalog to HiveClientImpl
## What changes were proposed in this pull request?
This removes the special case that `alterPartitions` call from `HiveExternalCatalog` can reset the current database in the hive client as a side effect.
## How was this patch tested?
(Please explain how this patch was tested. E.g. unit tests, integration tests, manual tests)
(If this patch involves UI changes, please attach a screenshot; otherwise, remove this)
Please review http://spark.apache.org/contributing.html before opening a pull request.
Author: Feng Liu <fe...@databricks.com>
Closes #20564 from liufengdb/move.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fba01b9a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fba01b9a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fba01b9a
Branch: refs/heads/master
Commit: fba01b9a65e5d9438d35da0bd807c179ba741911
Parents: 0c66fe4
Author: Feng Liu <fe...@databricks.com>
Authored: Mon Feb 12 14:58:31 2018 -0800
Committer: gatorsmile <ga...@gmail.com>
Committed: Mon Feb 12 14:58:31 2018 -0800
----------------------------------------------------------------------
.../spark/sql/hive/HiveExternalCatalog.scala | 5 ----
.../spark/sql/hive/client/HiveClientImpl.scala | 26 +++++++++++++++-----
2 files changed, 20 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/fba01b9a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
index 3b8a8ca..1ee1d57 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
@@ -1107,11 +1107,6 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
}
}
- // Note: Before altering table partitions in Hive, you *must* set the current database
- // to the one that contains the table of interest. Otherwise you will end up with the
- // most helpful error message ever: "Unable to alter partition. alter is not possible."
- // See HIVE-2742 for more detail.
- client.setCurrentDatabase(db)
client.alterPartitions(db, table, withStatsProps)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/fba01b9a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
index 6c0f414..c223f51 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
@@ -291,14 +291,18 @@ private[hive] class HiveClientImpl(
state.err = stream
}
- override def setCurrentDatabase(databaseName: String): Unit = withHiveState {
- if (databaseExists(databaseName)) {
- state.setCurrentDatabase(databaseName)
+ private def setCurrentDatabaseRaw(db: String): Unit = {
+ if (databaseExists(db)) {
+ state.setCurrentDatabase(db)
} else {
- throw new NoSuchDatabaseException(databaseName)
+ throw new NoSuchDatabaseException(db)
}
}
+ override def setCurrentDatabase(databaseName: String): Unit = withHiveState {
+ setCurrentDatabaseRaw(databaseName)
+ }
+
override def createDatabase(
database: CatalogDatabase,
ignoreIfExists: Boolean): Unit = withHiveState {
@@ -598,8 +602,18 @@ private[hive] class HiveClientImpl(
db: String,
table: String,
newParts: Seq[CatalogTablePartition]): Unit = withHiveState {
- val hiveTable = toHiveTable(getTable(db, table), Some(userName))
- shim.alterPartitions(client, table, newParts.map { p => toHivePartition(p, hiveTable) }.asJava)
+ // Note: Before altering table partitions in Hive, you *must* set the current database
+ // to the one that contains the table of interest. Otherwise you will end up with the
+ // most helpful error message ever: "Unable to alter partition. alter is not possible."
+ // See HIVE-2742 for more detail.
+ val original = state.getCurrentDatabase
+ try {
+ setCurrentDatabaseRaw(db)
+ val hiveTable = toHiveTable(getTable(db, table), Some(userName))
+ shim.alterPartitions(client, table, newParts.map { toHivePartition(_, hiveTable) }.asJava)
+ } finally {
+ state.setCurrentDatabase(original)
+ }
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org