You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2018/01/19 14:47:28 UTC
spark git commit: [SPARK-23000][TEST] Keep Derby DB Location
Unchanged After Session Cloning
Repository: spark
Updated Branches:
refs/heads/master e1c33b6cd -> 6c39654ef
[SPARK-23000][TEST] Keep Derby DB Location Unchanged After Session Cloning
## What changes were proposed in this pull request?
After session cloning in `TestHive`, the conf of the singleton SparkContext for derby DB location is changed to a new directory. The new directory is created in `HiveUtils.newTemporaryConfiguration(useInMemoryDerby = false)`.
This PR is to keep the conf value of `ConfVars.METASTORECONNECTURLKEY.varname` unchanged during the session clone.
## How was this patch tested?
The issue can be reproduced by the command:
> build/sbt -Phive "hive/test-only org.apache.spark.sql.hive.HiveSessionStateSuite org.apache.spark.sql.hive.DataSourceWithHiveMetastoreCatalogSuite"
Also added a test case.
Author: gatorsmile <ga...@gmail.com>
Closes #20328 from gatorsmile/fixTestFailure.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6c39654e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6c39654e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6c39654e
Branch: refs/heads/master
Commit: 6c39654efcb2aa8cb4d082ab7277a6fa38fb48e4
Parents: e1c33b6
Author: gatorsmile <ga...@gmail.com>
Authored: Fri Jan 19 22:47:18 2018 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Fri Jan 19 22:47:18 2018 +0800
----------------------------------------------------------------------
.../org/apache/spark/sql/SessionStateSuite.scala | 5 +----
.../org/apache/spark/sql/hive/test/TestHive.scala | 8 +++++++-
.../spark/sql/hive/HiveSessionStateSuite.scala | 16 +++++++++++++---
3 files changed, 21 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/6c39654e/sql/core/src/test/scala/org/apache/spark/sql/SessionStateSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SessionStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SessionStateSuite.scala
index 5d75f58..4efae4c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SessionStateSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SessionStateSuite.scala
@@ -17,8 +17,6 @@
package org.apache.spark.sql
-import org.scalatest.BeforeAndAfterAll
-import org.scalatest.BeforeAndAfterEach
import scala.collection.mutable.ArrayBuffer
import org.apache.spark.SparkFunSuite
@@ -28,8 +26,7 @@ import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.execution.QueryExecution
import org.apache.spark.sql.util.QueryExecutionListener
-class SessionStateSuite extends SparkFunSuite
- with BeforeAndAfterEach with BeforeAndAfterAll {
+class SessionStateSuite extends SparkFunSuite {
/**
* A shared SparkSession for all tests in this suite. Make sure you reset any changes to this
http://git-wip-us.apache.org/repos/asf/spark/blob/6c39654e/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
index b6be00d..c84131f 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
@@ -180,7 +180,13 @@ private[hive] class TestHiveSparkSession(
ConfVars.METASTORE_INTEGER_JDO_PUSHDOWN.varname -> "true",
// scratch directory used by Hive's metastore client
ConfVars.SCRATCHDIR.varname -> TestHiveContext.makeScratchDir().toURI.toString,
- ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY.varname -> "1")
+ ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY.varname -> "1") ++
+ // After session cloning, the JDBC connect string for a JDBC metastore should not be changed.
+ existingSharedState.map { state =>
+ val connKey =
+ state.sparkContext.hadoopConfiguration.get(ConfVars.METASTORECONNECTURLKEY.varname)
+ ConfVars.METASTORECONNECTURLKEY.varname -> connKey
+ }
metastoreTempConf.foreach { case (k, v) =>
sc.hadoopConfiguration.set(k, v)
http://git-wip-us.apache.org/repos/asf/spark/blob/6c39654e/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSessionStateSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSessionStateSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSessionStateSuite.scala
index f7da3c4..ecc09cd 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSessionStateSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSessionStateSuite.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.hive
-import org.scalatest.BeforeAndAfterEach
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars
import org.apache.spark.sql._
import org.apache.spark.sql.hive.test.TestHiveSingleton
@@ -25,8 +25,7 @@ import org.apache.spark.sql.hive.test.TestHiveSingleton
/**
* Run all tests from `SessionStateSuite` with a Hive based `SessionState`.
*/
-class HiveSessionStateSuite extends SessionStateSuite
- with TestHiveSingleton with BeforeAndAfterEach {
+class HiveSessionStateSuite extends SessionStateSuite with TestHiveSingleton {
override def beforeAll(): Unit = {
// Reuse the singleton session
@@ -39,4 +38,15 @@ class HiveSessionStateSuite extends SessionStateSuite
activeSession = null
super.afterAll()
}
+
+ test("Clone then newSession") {
+ val sparkSession = hiveContext.sparkSession
+ val conf = sparkSession.sparkContext.hadoopConfiguration
+ val oldValue = conf.get(ConfVars.METASTORECONNECTURLKEY.varname)
+ sparkSession.cloneSession()
+ sparkSession.sharedState.externalCatalog.client.newSession()
+ val newValue = conf.get(ConfVars.METASTORECONNECTURLKEY.varname)
+ assert(oldValue == newValue,
+ "cloneSession and then newSession should not affect the Derby directory")
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org