You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kg...@apache.org on 2018/01/26 12:34:31 UTC
hive git commit: HIVE-18383: Qtests: running all cases from
TestNegativeCliDriver results in OOMs (Zoltan Haindrich reviewed by Ashutosh
Chauhan)
Repository: hive
Updated Branches:
refs/heads/master c9a5286e7 -> 8de3334c2
HIVE-18383: Qtests: running all cases from TestNegativeCliDriver results in OOMs (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8de3334c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8de3334c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8de3334c
Branch: refs/heads/master
Commit: 8de3334c2ace3da97bca8dce0536e28f36367d84
Parents: c9a5286
Author: Zoltan Haindrich <ki...@rxd.hu>
Authored: Fri Jan 26 13:17:00 2018 +0100
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Fri Jan 26 13:17:00 2018 +0100
----------------------------------------------------------------------
.../hive/cli/control/AbstractCliConfig.java | 3 +-
.../org/apache/hadoop/hive/ql/QTestUtil.java | 11 ++++--
.../hadoop/hive/ql/session/SessionState.java | 38 +++++++++++++-------
3 files changed, 36 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/8de3334c/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
index 64db08a..cbba779 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCliConfig.java
@@ -30,6 +30,7 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Properties;
import java.util.Set;
+import java.util.TreeSet;
import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils;
@@ -237,7 +238,7 @@ public abstract class AbstractCliConfig {
File queryDir = new File(queryDirectory);
// dedup file list
- Set<File> testFiles = new LinkedHashSet<>();
+ Set<File> testFiles = new TreeSet<>();
if (queryFile != null && !queryFile.equals("")) {
// The user may have passed a list of files - comma separated
for (String qFile : TEST_SPLITTER.split(queryFile)) {
http://git-wip-us.apache.org/repos/asf/hive/blob/8de3334c/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 3f377f9..0b0b7ee 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -198,7 +198,6 @@ public class QTestUtil {
private MiniLlapCluster llapCluster = null;
private String hadoopVer = null;
private QTestSetup setup = null;
- private TezSessionState tezSessionState = null;
private SparkSession sparkSession = null;
private boolean isSessionStateStarted = false;
private static final String javaVersion = getJavaVersion();
@@ -1192,7 +1191,7 @@ public class QTestUtil {
boolean canReuseSession = !qNoSessionReuseQuerySet.contains(tname);
if (oldSs != null && canReuseSession && clusterType.getCoreClusterType() == CoreClusterType.TEZ) {
// Copy the tezSessionState from the old CliSessionState.
- tezSessionState = oldSs.getTezSession();
+ TezSessionState tezSessionState = oldSs.getTezSession();
oldSs.setTezSession(null);
ss.setTezSession(tezSessionState);
oldSs.close();
@@ -1208,6 +1207,9 @@ public class QTestUtil {
if (oldSs != null && oldSs.out != null && oldSs.out != System.out) {
oldSs.out.close();
}
+ if (oldSs != null) {
+ oldSs.close();
+ }
SessionState.start(ss);
cliDriver = new CliDriver();
@@ -1237,7 +1239,7 @@ public class QTestUtil {
SessionState oldSs = SessionState.get();
if (oldSs != null && canReuseSession && clusterType.getCoreClusterType() == CoreClusterType.TEZ) {
// Copy the tezSessionState from the old CliSessionState.
- tezSessionState = oldSs.getTezSession();
+ TezSessionState tezSessionState = oldSs.getTezSession();
ss.setTezSession(tezSessionState);
oldSs.setTezSession(null);
oldSs.close();
@@ -1252,6 +1254,9 @@ public class QTestUtil {
if (oldSs != null && oldSs.out != null && oldSs.out != System.out) {
oldSs.out.close();
}
+ if (oldSs != null) {
+ oldSs.close();
+ }
SessionState.start(ss);
isSessionStateStarted = true;
http://git-wip-us.apache.org/repos/asf/hive/blob/8de3334c/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 8af19d8..d6083e6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -561,7 +561,9 @@ public class SessionState {
public static void endStart(SessionState startSs)
throws CancellationException, InterruptedException {
- if (startSs.tezSessionState == null) return;
+ if (startSs.tezSessionState == null) {
+ return;
+ }
startSs.tezSessionState.endOpen();
}
@@ -617,7 +619,9 @@ public class SessionState {
}
String engine = HiveConf.getVar(startSs.getConf(), HiveConf.ConfVars.HIVE_EXECUTION_ENGINE);
- if (!engine.equals("tez") || startSs.isHiveServerQuery) return;
+ if (!engine.equals("tez") || startSs.isHiveServerQuery) {
+ return;
+ }
try {
if (startSs.tezSessionState == null) {
@@ -857,7 +861,7 @@ public class SessionState {
fs.cancelDeleteOnExit(path);
fs.delete(path, true);
LOG.info("Deleted directory: {} on fs with scheme {}", path, fs.getScheme());
- } catch (IOException e) {
+ } catch (IllegalArgumentException | UnsupportedOperationException | IOException e) {
LOG.error("Failed to delete path at {} on fs with scheme {}", path,
(fs == null ? "Unknown-null" : fs.getScheme()), e);
}
@@ -1275,7 +1279,9 @@ public class SessionState {
*/
public void loadAuxJars() throws IOException {
String[] jarPaths = StringUtils.split(sessionConf.getAuxJars(), ',');
- if (ArrayUtils.isEmpty(jarPaths)) return;
+ if (ArrayUtils.isEmpty(jarPaths)) {
+ return;
+ }
URLClassLoader currentCLoader =
(URLClassLoader) SessionState.get().getConf().getClassLoader();
@@ -1708,7 +1714,9 @@ public class SessionState {
}
registry.clear();
- if (txnMgr != null) txnMgr.closeTxnManager();
+ if (txnMgr != null) {
+ txnMgr.closeTxnManager();
+ }
JavaUtils.closeClassLoadersTo(sessionConf.getClassLoader(), parentLoader);
File resourceDir =
new File(getConf().getVar(HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR));
@@ -1750,12 +1758,16 @@ public class SessionState {
boolean isLocalMetastore =
HiveConfUtil.isEmbeddedMetaStore(sessionConf.getVar(HiveConf.ConfVars.METASTOREURIS));
if (isLocalMetastore) {
- if (sessionConf.getVar(ConfVars.METASTORE_RAW_STORE_IMPL)
- .equals(ObjectStore.class.getName()) ||
- sessionConf.getVar(ConfVars.METASTORE_RAW_STORE_IMPL)
- .equals(CachedStore.class.getName()) && sessionConf
- .getVar(ConfVars.METASTORE_CACHED_RAW_STORE_IMPL)
- .equals(ObjectStore.class.getName())) {
+
+ String rawStoreImpl = sessionConf.getVar(ConfVars.METASTORE_RAW_STORE_IMPL);
+ String realStoreImpl;
+ if (rawStoreImpl.equals(CachedStore.class.getName())) {
+ realStoreImpl = sessionConf.getVar(ConfVars.METASTORE_CACHED_RAW_STORE_IMPL);
+ } else {
+ realStoreImpl = rawStoreImpl;
+ }
+ Class<?> clazz = Class.forName(realStoreImpl);
+ if (ObjectStore.class.isAssignableFrom(clazz)) {
ObjectStore.unCacheDataNucleusClassLoaders();
}
}
@@ -1823,7 +1835,9 @@ public class SessionState {
/** Called from TezTask to attach a TezSession to use to the threadlocal. Ugly pattern... */
public void setTezSession(TezSessionState session) {
- if (tezSessionState == session) return; // The same object.
+ if (tezSessionState == session) {
+ return; // The same object.
+ }
if (tezSessionState != null) {
tezSessionState.markFree();
tezSessionState.setKillQuery(null);