You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2017/11/10 11:01:08 UTC
spark git commit: [SPARK-22487][SQL][HIVE] Remove the unused
HIVE_EXECUTION_VERSION property
Repository: spark
Updated Branches:
refs/heads/master 0025ddeb1 -> 28ab5bf59
[SPARK-22487][SQL][HIVE] Remove the unused HIVE_EXECUTION_VERSION property
## What changes were proposed in this pull request?
At the beginning https://github.com/apache/spark/pull/2843 added `spark.sql.hive.version` to reveal underlying hive version for jdbc connections. For some time afterwards, it was used as a version identifier for the execution hive client.
Actually there is no hive client for executions in spark now and there are no usages of HIVE_EXECUTION_VERSION found in whole spark project. HIVE_EXECUTION_VERSION is set by `spark.sql.hive.version`, which is still set internally in some places or by users, this may confuse developers and users with HIVE_METASTORE_VERSION(spark.sql.hive.metastore.version).
It might better to be removed.
## How was this patch tested?
modify some existing ut
cc cloud-fan gatorsmile
Author: Kent Yao <ya...@hotmail.com>
Closes #19712 from yaooqinn/SPARK-22487.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/28ab5bf5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/28ab5bf5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/28ab5bf5
Branch: refs/heads/master
Commit: 28ab5bf59766096bb7e1bdab32b05cb5f0799a48
Parents: 0025dde
Author: Kent Yao <ya...@hotmail.com>
Authored: Fri Nov 10 12:01:02 2017 +0100
Committer: Wenchen Fan <we...@databricks.com>
Committed: Fri Nov 10 12:01:02 2017 +0100
----------------------------------------------------------------------
.../sql/hive/thriftserver/SparkSQLEnv.scala | 1 -
.../thriftserver/SparkSQLSessionManager.scala | 1 -
.../thriftserver/HiveThriftServer2Suites.scala | 27 +++++---------------
.../org/apache/spark/sql/hive/HiveUtils.scala | 25 ++++++++----------
.../spark/sql/hive/client/VersionsSuite.scala | 4 +--
5 files changed, 19 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/28ab5bf5/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala
----------------------------------------------------------------------
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala
index 01c4eb1..5db93b2 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLEnv.scala
@@ -55,7 +55,6 @@ private[hive] object SparkSQLEnv extends Logging {
metadataHive.setOut(new PrintStream(System.out, true, "UTF-8"))
metadataHive.setInfo(new PrintStream(System.err, true, "UTF-8"))
metadataHive.setError(new PrintStream(System.err, true, "UTF-8"))
- sparkSession.conf.set("spark.sql.hive.version", HiveUtils.hiveExecutionVersion)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/28ab5bf5/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLSessionManager.scala
----------------------------------------------------------------------
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLSessionManager.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLSessionManager.scala
index 7adaafe..00920c2 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLSessionManager.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLSessionManager.scala
@@ -77,7 +77,6 @@ private[hive] class SparkSQLSessionManager(hiveServer: HiveServer2, sqlContext:
} else {
sqlContext.newSession()
}
- ctx.setConf("spark.sql.hive.version", HiveUtils.hiveExecutionVersion)
if (sessionConf != null && sessionConf.containsKey("use:database")) {
ctx.sql(s"use ${sessionConf.get("use:database")}")
}
http://git-wip-us.apache.org/repos/asf/spark/blob/28ab5bf5/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
----------------------------------------------------------------------
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
index 4997d7f..b80596f 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala
@@ -155,10 +155,10 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
test("Checks Hive version") {
withJdbcStatement() { statement =>
- val resultSet = statement.executeQuery("SET spark.sql.hive.version")
+ val resultSet = statement.executeQuery("SET spark.sql.hive.metastore.version")
resultSet.next()
- assert(resultSet.getString(1) === "spark.sql.hive.version")
- assert(resultSet.getString(2) === HiveUtils.hiveExecutionVersion)
+ assert(resultSet.getString(1) === "spark.sql.hive.metastore.version")
+ assert(resultSet.getString(2) === HiveUtils.builtinHiveVersion)
}
}
@@ -521,20 +521,7 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest {
conf += resultSet.getString(1) -> resultSet.getString(2)
}
- assert(conf.get("spark.sql.hive.version") === Some("1.2.1"))
- }
- }
-
- test("Checks Hive version via SET") {
- withJdbcStatement() { statement =>
- val resultSet = statement.executeQuery("SET")
-
- val conf = mutable.Map.empty[String, String]
- while (resultSet.next()) {
- conf += resultSet.getString(1) -> resultSet.getString(2)
- }
-
- assert(conf.get("spark.sql.hive.version") === Some("1.2.1"))
+ assert(conf.get("spark.sql.hive.metastore.version") === Some("1.2.1"))
}
}
@@ -721,10 +708,10 @@ class HiveThriftHttpServerSuite extends HiveThriftJdbcTest {
test("Checks Hive version") {
withJdbcStatement() { statement =>
- val resultSet = statement.executeQuery("SET spark.sql.hive.version")
+ val resultSet = statement.executeQuery("SET spark.sql.hive.metastore.version")
resultSet.next()
- assert(resultSet.getString(1) === "spark.sql.hive.version")
- assert(resultSet.getString(2) === HiveUtils.hiveExecutionVersion)
+ assert(resultSet.getString(1) === "spark.sql.hive.metastore.version")
+ assert(resultSet.getString(2) === HiveUtils.builtinHiveVersion)
}
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/28ab5bf5/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
index 80b9a3d..d8e08f1 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala
@@ -58,28 +58,23 @@ private[spark] object HiveUtils extends Logging {
}
/** The version of hive used internally by Spark SQL. */
- val hiveExecutionVersion: String = "1.2.1"
+ val builtinHiveVersion: String = "1.2.1"
val HIVE_METASTORE_VERSION = buildConf("spark.sql.hive.metastore.version")
.doc("Version of the Hive metastore. Available options are " +
- s"<code>0.12.0</code> through <code>$hiveExecutionVersion</code>.")
+ s"<code>0.12.0</code> through <code>2.1.1</code>.")
.stringConf
- .createWithDefault(hiveExecutionVersion)
-
- val HIVE_EXECUTION_VERSION = buildConf("spark.sql.hive.version")
- .doc("Version of Hive used internally by Spark SQL.")
- .stringConf
- .createWithDefault(hiveExecutionVersion)
+ .createWithDefault(builtinHiveVersion)
val HIVE_METASTORE_JARS = buildConf("spark.sql.hive.metastore.jars")
.doc(s"""
| Location of the jars that should be used to instantiate the HiveMetastoreClient.
| This property can be one of three options: "
| 1. "builtin"
- | Use Hive ${hiveExecutionVersion}, which is bundled with the Spark assembly when
+ | Use Hive ${builtinHiveVersion}, which is bundled with the Spark assembly when
| <code>-Phive</code> is enabled. When this option is chosen,
| <code>spark.sql.hive.metastore.version</code> must be either
- | <code>${hiveExecutionVersion}</code> or not defined.
+ | <code>${builtinHiveVersion}</code> or not defined.
| 2. "maven"
| Use Hive jars of specified version downloaded from Maven repositories.
| 3. A classpath in the standard format for both Hive and Hadoop.
@@ -259,9 +254,9 @@ private[spark] object HiveUtils extends Logging {
protected[hive] def newClientForExecution(
conf: SparkConf,
hadoopConf: Configuration): HiveClientImpl = {
- logInfo(s"Initializing execution hive, version $hiveExecutionVersion")
+ logInfo(s"Initializing execution hive, version $builtinHiveVersion")
val loader = new IsolatedClientLoader(
- version = IsolatedClientLoader.hiveVersion(hiveExecutionVersion),
+ version = IsolatedClientLoader.hiveVersion(builtinHiveVersion),
sparkConf = conf,
execJars = Seq.empty,
hadoopConf = hadoopConf,
@@ -297,12 +292,12 @@ private[spark] object HiveUtils extends Logging {
val metaVersion = IsolatedClientLoader.hiveVersion(hiveMetastoreVersion)
val isolatedLoader = if (hiveMetastoreJars == "builtin") {
- if (hiveExecutionVersion != hiveMetastoreVersion) {
+ if (builtinHiveVersion != hiveMetastoreVersion) {
throw new IllegalArgumentException(
"Builtin jars can only be used when hive execution version == hive metastore version. " +
- s"Execution: $hiveExecutionVersion != Metastore: $hiveMetastoreVersion. " +
+ s"Execution: $builtinHiveVersion != Metastore: $hiveMetastoreVersion. " +
"Specify a vaild path to the correct hive jars using $HIVE_METASTORE_JARS " +
- s"or change ${HIVE_METASTORE_VERSION.key} to $hiveExecutionVersion.")
+ s"or change ${HIVE_METASTORE_VERSION.key} to $builtinHiveVersion.")
}
// We recursively find all jars in the class loader chain,
http://git-wip-us.apache.org/repos/asf/spark/blob/28ab5bf5/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
index edb9a9f..9ed39cc 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
@@ -73,7 +73,7 @@ class VersionsSuite extends SparkFunSuite with Logging {
}
test("success sanity check") {
- val badClient = buildClient(HiveUtils.hiveExecutionVersion, new Configuration())
+ val badClient = buildClient(HiveUtils.builtinHiveVersion, new Configuration())
val db = new CatalogDatabase("default", "desc", new URI("loc"), Map())
badClient.createDatabase(db, ignoreIfExists = true)
}
@@ -81,7 +81,7 @@ class VersionsSuite extends SparkFunSuite with Logging {
test("hadoop configuration preserved") {
val hadoopConf = new Configuration()
hadoopConf.set("test", "success")
- val client = buildClient(HiveUtils.hiveExecutionVersion, hadoopConf)
+ val client = buildClient(HiveUtils.builtinHiveVersion, hadoopConf)
assert("success" === client.getConf("test", null))
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org