You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2019/04/04 05:53:01 UTC
[spark] branch master updated: [SPARK-27349][SQL] Dealing with
TimeVars removed in Hive 2.x
This is an automated email from the ASF dual-hosted git repository.
lixiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 1d95dea [SPARK-27349][SQL] Dealing with TimeVars removed in Hive 2.x
1d95dea is described below
commit 1d95dea30788b9f64c5e304d908b85936aafb238
Author: Yuming Wang <yu...@ebay.com>
AuthorDate: Wed Apr 3 22:52:37 2019 -0700
[SPARK-27349][SQL] Dealing with TimeVars removed in Hive 2.x
## What changes were proposed in this pull request?
`hive.stats.jdbc.timeout` and `hive.stats.retries.wait` were removed by [HIVE-12164](https://issues.apache.org/jira/browse/HIVE-12164).
This pr to deal with this change.
## How was this patch tested?
unit tests
Closes #24277 from wangyum/SPARK-27349.
Authored-by: Yuming Wang <yu...@ebay.com>
Signed-off-by: gatorsmile <ga...@gmail.com>
---
.../scala/org/apache/spark/sql/hive/HiveUtils.scala | 17 +++++++++++++----
.../org/apache/spark/sql/hive/HiveUtilsSuite.scala | 20 ++++++++++++++++++++
2 files changed, 33 insertions(+), 4 deletions(-)
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
index a7f40c6..01a503d 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
@@ -194,7 +194,7 @@ private[spark] object HiveUtils extends Logging {
//
// Here we enumerate all time `ConfVar`s and convert their values to numeric strings according
// to their output time units.
- Seq(
+ val commonTimeVars = Seq(
ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY -> TimeUnit.SECONDS,
ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT -> TimeUnit.SECONDS,
ConfVars.METASTORE_CLIENT_SOCKET_LIFETIME -> TimeUnit.SECONDS,
@@ -207,8 +207,6 @@ private[spark] object HiveUtils extends Logging {
ConfVars.METASTORE_AGGREGATE_STATS_CACHE_MAX_READER_WAIT -> TimeUnit.MILLISECONDS,
ConfVars.HIVES_AUTO_PROGRESS_TIMEOUT -> TimeUnit.SECONDS,
ConfVars.HIVE_LOG_INCREMENTAL_PLAN_PROGRESS_INTERVAL -> TimeUnit.MILLISECONDS,
- ConfVars.HIVE_STATS_JDBC_TIMEOUT -> TimeUnit.SECONDS,
- ConfVars.HIVE_STATS_RETRIES_WAIT -> TimeUnit.MILLISECONDS,
ConfVars.HIVE_LOCK_SLEEP_BETWEEN_RETRIES -> TimeUnit.SECONDS,
ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT -> TimeUnit.MILLISECONDS,
ConfVars.HIVE_ZOOKEEPER_CONNECTION_BASESLEEPTIME -> TimeUnit.MILLISECONDS,
@@ -236,7 +234,18 @@ private[spark] object HiveUtils extends Logging {
ConfVars.SPARK_RPC_CLIENT_HANDSHAKE_TIMEOUT -> TimeUnit.MILLISECONDS
).map { case (confVar, unit) =>
confVar.varname -> HiveConf.getTimeVar(hadoopConf, confVar, unit).toString
- }.toMap
+ }
+
+ // The following configurations were removed by HIVE-12164(Hive 2.0)
+ val hardcodingTimeVars = Seq(
+ ("hive.stats.jdbc.timeout", "30s") -> TimeUnit.SECONDS,
+ ("hive.stats.retries.wait", "3000ms") -> TimeUnit.MILLISECONDS
+ ).map { case ((key, defaultValue), unit) =>
+ val value = hadoopConf.get(key, defaultValue)
+ key -> HiveConf.toTime(value, unit, unit).toString
+ }
+
+ (commonTimeVars ++ hardcodingTimeVars).toMap
}
/**
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala
index 303dd70..daf0664 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUtilsSuite.scala
@@ -17,6 +17,7 @@
package org.apache.spark.sql.hive
+import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hive.conf.HiveConf.ConfVars
import org.apache.spark.SparkConf
@@ -29,6 +30,12 @@ import org.apache.spark.util.ChildFirstURLClassLoader
class HiveUtilsSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
+ private def testFormatTimeVarsForHiveClient(key: String, value: String, expected: Long): Unit = {
+ val conf = new Configuration
+ conf.set(key, value)
+ assert(HiveUtils.formatTimeVarsForHiveClient(conf)(key) === expected.toString)
+ }
+
test("newTemporaryConfiguration overwrites listener configurations") {
Seq(true, false).foreach { useInMemoryDerby =>
val conf = HiveUtils.newTemporaryConfiguration(useInMemoryDerby)
@@ -61,4 +68,17 @@ class HiveUtilsSuite extends QueryTest with SQLTestUtils with TestHiveSingleton
Thread.currentThread().setContextClassLoader(contextClassLoader)
}
}
+
+ test("SPARK-27349: Dealing with TimeVars removed in Hive 2.x") {
+ // Test default value
+ val defaultConf = new Configuration
+ assert(HiveUtils.formatTimeVarsForHiveClient(defaultConf)("hive.stats.jdbc.timeout") === "30")
+ assert(HiveUtils.formatTimeVarsForHiveClient(defaultConf)("hive.stats.retries.wait") === "3000")
+
+ testFormatTimeVarsForHiveClient("hive.stats.jdbc.timeout", "40s", 40)
+ testFormatTimeVarsForHiveClient("hive.stats.jdbc.timeout", "1d", 1 * 24 * 60 * 60)
+
+ testFormatTimeVarsForHiveClient("hive.stats.retries.wait", "4000ms", 4000)
+ testFormatTimeVarsForHiveClient("hive.stats.retries.wait", "1d", 1 * 24 * 60 * 60 * 1000)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org