You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pv...@apache.org on 2018/10/09 11:48:03 UTC
hive git commit: HIVE-20711: Race Condition when Multi-Threading in
SessionState.createRootHDFSDir (Denys Kuzmenko,
reviewed by Antal Skinkovics and Peter Vary)
Repository: hive
Updated Branches:
refs/heads/master 1e048df35 -> 717a15b38
HIVE-20711: Race Condition when Multi-Threading in SessionState.createRootHDFSDir (Denys Kuzmenko, reviewed by Antal Skinkovics and Peter Vary)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/717a15b3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/717a15b3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/717a15b3
Branch: refs/heads/master
Commit: 717a15b383e67c635870ec0369e77454541b994a
Parents: 1e048df
Author: denys kuzmenko <dk...@cloudera.com>
Authored: Tue Oct 9 09:47:58 2018 +0200
Committer: Peter Vary <pv...@cloudera.com>
Committed: Tue Oct 9 09:47:58 2018 +0200
----------------------------------------------------------------------
.../ql/exec/spark/TestSparkSessionTimeout.java | 27 +-------------------
.../apache/hadoop/hive/ql/exec/Utilities.java | 14 +++++++---
2 files changed, 11 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/717a15b3/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java
index c887297..d8dd80a 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java
@@ -68,32 +68,7 @@ public class TestSparkSessionTimeout {
HiveConf conf = new HiveConf();
conf.setBoolVar(HiveConf.ConfVars.SPARK_OPTIMIZE_SHUFFLE_SERDE, false);
conf.set("spark.local.dir", Paths.get(System.getProperty("test.tmp.dir"),
- "TestSparkSessionTimeout-testMultiSparkSessionTimeout-local-dir").toString());
-
- SessionState.start(conf);
-
- runTestSparkSessionTimeout(conf);
- return null;
- }));
- }
- for (Future<Void> future : futures) {
- future.get();
- }
- }
-
- @Test
- public void testMultiSparkSessionTimeout() throws ExecutionException, InterruptedException {
- List<Future<Void>> futures = new ArrayList<>();
- ExecutorService es = Executors.newFixedThreadPool(10);
- for (int i = 0; i < 10; i++) {
- futures.add(es.submit(() -> {
- String confDir = "../../data/conf/spark/local/hive-site.xml";
- HiveConf.setHiveSiteLocation(new File(confDir).toURI().toURL());
-
- HiveConf conf = new HiveConf();
- conf.setBoolVar(HiveConf.ConfVars.SPARK_OPTIMIZE_SHUFFLE_SERDE, false);
- conf.set("spark.local.dir", Paths.get(System.getProperty("test.tmp.dir"),
- "TestSparkSessionTimeout-testMultiSparkSessionTimeout-local-dir").toString());
+ "TestSparkSessionTimeout-testMultiSessionSparkSessionTimeout-local-dir").toString());
SessionState.start(conf);
http://git-wip-us.apache.org/repos/asf/hive/blob/717a15b3/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 76a30eb..10aa94e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -255,6 +255,9 @@ public final class Utilities {
public static Random randGen = new Random();
+ private static final Object INPUT_SUMMARY_LOCK = new Object();
+ private static final Object ROOT_HDFS_DIR_LOCK = new Object();
+
/**
* ReduceField:
* KEY: record key
@@ -2317,8 +2320,6 @@ public final class Utilities {
}
}
- private static final Object INPUT_SUMMARY_LOCK = new Object();
-
/**
* Returns the maximum number of executors required to get file information from several input locations.
* It checks whether HIVE_EXEC_INPUT_LISTING_MAX_THREADS or DEPRECATED_MAPRED_DFSCLIENT_PARALLELISM_MAX are > 1
@@ -4463,11 +4464,16 @@ public final class Utilities {
public static void ensurePathIsWritable(Path rootHDFSDirPath, HiveConf conf) throws IOException {
FsPermission writableHDFSDirPermission = new FsPermission((short)00733);
FileSystem fs = rootHDFSDirPath.getFileSystem(conf);
+
if (!fs.exists(rootHDFSDirPath)) {
- Utilities.createDirsWithPermission(conf, rootHDFSDirPath, writableHDFSDirPermission, true);
+ synchronized (ROOT_HDFS_DIR_LOCK) {
+ if (!fs.exists(rootHDFSDirPath)) {
+ Utilities.createDirsWithPermission(conf, rootHDFSDirPath, writableHDFSDirPermission, true);
+ }
+ }
}
FsPermission currentHDFSDirPermission = fs.getFileStatus(rootHDFSDirPath).getPermission();
- if (rootHDFSDirPath != null && rootHDFSDirPath.toUri() != null) {
+ if (rootHDFSDirPath.toUri() != null) {
String schema = rootHDFSDirPath.toUri().getScheme();
LOG.debug("HDFS dir: " + rootHDFSDirPath + " with schema " + schema + ", permission: " +
currentHDFSDirPermission);