You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by st...@apache.org on 2018/05/29 17:07:09 UTC
hive git commit: HIVE-19716: Set spark.local.dir for a few more HoS
integration tests (Sahil Takiar, reviewed by Peter Vary)
Repository: hive
Updated Branches:
refs/heads/master 47e85b73d -> 068d007b8
HIVE-19716: Set spark.local.dir for a few more HoS integration tests (Sahil Takiar, reviewed by Peter Vary)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/068d007b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/068d007b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/068d007b
Branch: refs/heads/master
Commit: 068d007b8874f9eecbf30bb2739831ad9114107a
Parents: 47e85b7
Author: Sahil Takiar <ta...@gmail.com>
Authored: Fri May 25 11:52:18 2018 -0700
Committer: Sahil Takiar <st...@cloudera.com>
Committed: Tue May 29 12:05:38 2018 -0500
----------------------------------------------------------------------
.../hive/ql/exec/spark/TestSparkStatistics.java | 3 +++
.../jdbc/TestJdbcWithLocalClusterSpark.java | 3 +++
...stMultiSessionsHS2WithLocalClusterSpark.java | 3 +++
.../session/TestSparkSessionManagerImpl.java | 24 ++++++++++++--------
.../hive/spark/client/TestSparkClient.java | 4 ++++
5 files changed, 27 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/068d007b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkStatistics.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkStatistics.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkStatistics.java
index be3b501..4413161 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkStatistics.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkStatistics.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hive.ql.session.SessionState;
import org.junit.Assert;
import org.junit.Test;
+import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@@ -47,6 +48,8 @@ public class TestSparkStatistics {
conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
conf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "spark");
conf.set("spark.master", "local-cluster[1,2,1024]");
+ conf.set("spark.local.dir", Paths.get(System.getProperty("test.tmp.dir"),
+ "TestSparkStatistics-local-dir").toString());
SessionState.start(conf);
http://git-wip-us.apache.org/repos/asf/hive/blob/068d007b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithLocalClusterSpark.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithLocalClusterSpark.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithLocalClusterSpark.java
index 2a4da20..fe8a32f 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithLocalClusterSpark.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithLocalClusterSpark.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
@@ -76,6 +77,8 @@ public class TestJdbcWithLocalClusterSpark {
// Spark requires Hive to support Hadoop3 first then Spark can start
// working on Hadoop3 support. Remove this after Spark supports Hadoop3.
conf.set("dfs.client.datanode-restart.timeout", "30");
+ conf.set("spark.local.dir", Paths.get(System.getProperty("test.tmp.dir"),
+ "TestJdbcWithLocalClusterSpark-local-dir").toString());
return conf;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/068d007b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestMultiSessionsHS2WithLocalClusterSpark.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestMultiSessionsHS2WithLocalClusterSpark.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestMultiSessionsHS2WithLocalClusterSpark.java
index 9f72e51..79d56f5 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestMultiSessionsHS2WithLocalClusterSpark.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestMultiSessionsHS2WithLocalClusterSpark.java
@@ -18,6 +18,7 @@
package org.apache.hive.jdbc;
+import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
@@ -85,6 +86,8 @@ public class TestMultiSessionsHS2WithLocalClusterSpark {
// Spark requires Hive to support Hadoop3 first then Spark can start
// working on Hadoop3 support. Remove this after Spark supports Hadoop3.
conf.set("dfs.client.datanode-restart.timeout", "30");
+ conf.set("spark.local.dir", Paths.get(System.getProperty("test.tmp.dir"),
+ "TestMultiSessionsHS2WithLocalClusterSpark-local-dir").toString());
return conf;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/068d007b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java
index 5924b8b..15756da 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/spark/session/TestSparkSessionManagerImpl.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.util.StringUtils;
import org.junit.Test;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
@@ -49,8 +50,7 @@ public class TestSparkSessionManagerImpl {
/** Tests CLI scenario where we get a single session and use it multiple times. */
@Test
public void testSingleSessionMultipleUse() throws Exception {
- HiveConf conf = new HiveConf();
- conf.set("spark.master", "local");
+ HiveConf conf = getHiveConf();
SparkSessionManager sessionManager = SparkSessionManagerImpl.getInstance();
SparkSession sparkSession1 = sessionManager.getSession(null, conf, true);
@@ -76,8 +76,7 @@ public class TestSparkSessionManagerImpl {
// Shutdown existing session manager
sessionManagerHS2.shutdown();
- HiveConf hiveConf = new HiveConf();
- hiveConf.set("spark.master", "local");
+ HiveConf hiveConf = getHiveConf();
sessionManagerHS2.setup(hiveConf);
@@ -110,8 +109,7 @@ public class TestSparkSessionManagerImpl {
*/
@Test
public void testForceConfCloning() throws Exception {
- HiveConf conf = new HiveConf();
- conf.set("spark.master", "local");
+ HiveConf conf = getHiveConf();
String sparkCloneConfiguration = HiveSparkClientFactory.SPARK_CLONE_CONFIGURATION;
// Clear the value of sparkCloneConfiguration
@@ -133,8 +131,7 @@ public class TestSparkSessionManagerImpl {
@Test
public void testGetHiveException() throws Exception {
- HiveConf conf = new HiveConf();
- conf.set("spark.master", "local");
+ HiveConf conf = getHiveConf();
SparkSessionManager ssm = SparkSessionManagerImpl.getInstance();
SparkSessionImpl ss = (SparkSessionImpl) ssm.getSession(
null, conf, true);
@@ -229,8 +226,7 @@ public class TestSparkSessionManagerImpl {
Random random = new Random(Thread.currentThread().getId());
String threadName = Thread.currentThread().getName();
System.out.println(threadName + " started.");
- HiveConf conf = new HiveConf();
- conf.set("spark.master", "local");
+ HiveConf conf = getHiveConf();
SparkSession prevSession = null;
SparkSession currentSession = null;
@@ -256,4 +252,12 @@ public class TestSparkSessionManagerImpl {
}
}
}
+
+ private HiveConf getHiveConf() {
+ HiveConf conf = new HiveConf();
+ conf.set("spark.master", "local");
+ conf.set("spark.local.dir", Paths.get(System.getProperty("test.tmp.dir"),
+ "TestSparkSessionManagerImpl-local-dir").toString());
+ return conf;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/068d007b/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java
----------------------------------------------------------------------
diff --git a/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java b/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java
index fdf882b..c134625 100644
--- a/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java
+++ b/spark-client/src/test/java/org/apache/hive/spark/client/TestSparkClient.java
@@ -37,6 +37,7 @@ import java.io.InputStream;
import java.io.PrintStream;
import java.io.Serializable;
import java.net.URI;
+import java.nio.file.Paths;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
@@ -86,6 +87,9 @@ public class TestSparkClient {
conf.put("spark.home", System.getProperty("spark.home"));
}
+ conf.put("spark.local.dir", Paths.get(System.getProperty("test.tmp.dir"),
+ "TestSparkClient-local-dir").toString());
+
return conf;
}