You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pv...@apache.org on 2018/10/19 10:00:57 UTC
hive git commit: HIVE-20742: SparkSessionManagerImpl maintenance
thread only cleans up session once (Antal Sinkovits reviewed by Sahil Takiar
and Peter Vary)
Repository: hive
Updated Branches:
refs/heads/master 3963c729f -> bd3c05d72
HIVE-20742: SparkSessionManagerImpl maintenance thread only cleans up session once (Antal Sinkovits reviewed by Sahil Takiar and Peter Vary)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bd3c05d7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bd3c05d7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bd3c05d7
Branch: refs/heads/master
Commit: bd3c05d72eb31b975a76b5ec2bdd7603f2ed82de
Parents: 3963c72
Author: Antal Sinkovits <as...@cloudera.com>
Authored: Fri Oct 19 11:59:56 2018 +0200
Committer: Peter Vary <pv...@cloudera.com>
Committed: Fri Oct 19 11:59:56 2018 +0200
----------------------------------------------------------------------
.../ql/exec/spark/TestSparkSessionTimeout.java | 30 +++++++++++++++-----
.../spark/session/SparkSessionManagerImpl.java | 1 +
2 files changed, 24 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/bd3c05d7/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java
index d8dd80a..7ede07d 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java
@@ -52,7 +52,7 @@ public class TestSparkSessionTimeout {
SessionState.start(conf);
- runTestSparkSessionTimeout(conf);
+ runTestSparkSessionTimeout(conf, 1);
}
@Test
@@ -72,7 +72,7 @@ public class TestSparkSessionTimeout {
SessionState.start(conf);
- runTestSparkSessionTimeout(conf);
+ runTestSparkSessionTimeout(conf, 1);
return null;
}));
}
@@ -81,7 +81,21 @@ public class TestSparkSessionTimeout {
}
}
- private void runTestSparkSessionTimeout(HiveConf conf) throws HiveException,
+ @Test
+ public void testSparkSessionMultipleTimeout() throws HiveException, InterruptedException, MalformedURLException {
+ String confDir = "../../data/conf/spark/standalone/hive-site.xml";
+ HiveConf.setHiveSiteLocation(new File(confDir).toURI().toURL());
+
+ HiveConf conf = new HiveConf();
+ conf.set("spark.local.dir", Paths.get(System.getProperty("test.tmp.dir"),
+ "TestSparkSessionTimeout-testSparkSessionMultipleTimeout-local-dir").toString());
+
+ SessionState.start(conf);
+
+ runTestSparkSessionTimeout(conf, 2);
+ }
+
+ private void runTestSparkSessionTimeout(HiveConf conf, int sleepRunIteration) throws HiveException,
InterruptedException {
conf.setVar(HiveConf.ConfVars.SPARK_SESSION_TIMEOUT, "5s");
conf.setVar(HiveConf.ConfVars.SPARK_SESSION_TIMEOUT_PERIOD, "1s");
@@ -104,12 +118,14 @@ public class TestSparkSessionTimeout {
Assert.assertEquals(0,
driver.run("select * from " + tableName + " order by col").getResponseCode());
- Thread.sleep(10000);
+ for (int i = 0; i < sleepRunIteration; i++) {
+ Thread.sleep(10000);
- Assert.assertFalse(sparkSession.isOpen());
+ Assert.assertFalse(sparkSession.isOpen());
- Assert.assertEquals(0,
- driver.run("select * from " + tableName + " order by col").getResponseCode());
+ Assert.assertEquals(0,
+ driver.run("select * from " + tableName + " order by col").getResponseCode());
+ }
} finally {
if (driver != null) {
Assert.assertEquals(0, driver.run("drop table if exists " + tableName).getResponseCode());
http://git-wip-us.apache.org/repos/asf/hive/blob/bd3c05d7/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionManagerImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionManagerImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionManagerImpl.java
index 8dae54d..e2f3a11 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionManagerImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionManagerImpl.java
@@ -121,6 +121,7 @@ public class SparkSessionManagerImpl implements SparkSessionManager {
// Open the session if it is closed.
if (!existingSession.isOpen() && doOpen) {
existingSession.open(conf);
+ createdSessions.add(existingSession);
}
return existingSession;
}