You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vi...@apache.org on 2014/09/30 20:36:57 UTC
svn commit: r1628512 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/exec/tez/
java/org/apache/hadoop/hive/ql/session/
test/org/apache/hadoop/hive/ql/exec/tez/
Author: vikram
Date: Tue Sep 30 18:36:56 2014
New Revision: 1628512
URL: http://svn.apache.org/r1628512
Log:
HIVE-7857: Hive query fails after Tez session times out (Vikram Dixit K, reviewed by Gunther Hagleitner)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java?rev=1628512&r1=1628511&r2=1628512&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java Tue Sep 30 18:36:56 2014
@@ -78,7 +78,7 @@ public class TezJobMonitor {
try {
for (TezSessionState s: TezSessionState.getOpenSessions()) {
System.err.println("Shutting down tez session.");
- TezSessionPoolManager.getInstance().close(s);
+ TezSessionPoolManager.getInstance().close(s, false);
}
} catch (Exception e) {
// ignore
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java?rev=1628512&r1=1628511&r2=1628512&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionPoolManager.java Tue Sep 30 18:36:56 2014
@@ -168,10 +168,10 @@ public class TezSessionPoolManager {
// session in the SessionState
}
- public void close(TezSessionState tezSessionState) throws Exception {
+ public void close(TezSessionState tezSessionState, boolean keepTmpDir) throws Exception {
LOG.info("Closing tez session default? " + tezSessionState.isDefault());
if (!tezSessionState.isDefault()) {
- tezSessionState.close(false);
+ tezSessionState.close(keepTmpDir);
}
}
@@ -262,24 +262,24 @@ public class TezSessionPoolManager {
}
if (session != null) {
- close(session);
+ close(session, false);
}
return getSession(conf, doOpen, forceCreate);
}
- public void closeAndOpen(TezSessionState sessionState, HiveConf conf)
+ public void closeAndOpen(TezSessionState sessionState, HiveConf conf, boolean keepTmpDir)
throws Exception {
- closeAndOpen(sessionState, conf, null);
+ closeAndOpen(sessionState, conf, null, keepTmpDir);
}
public void closeAndOpen(TezSessionState sessionState, HiveConf conf,
- String[] additionalFiles) throws Exception {
+ String[] additionalFiles, boolean keepTmpDir) throws Exception {
HiveConf sessionConf = sessionState.getConf();
if (sessionConf != null && sessionConf.get("tez.queue.name") != null) {
conf.set("tez.queue.name", sessionConf.get("tez.queue.name"));
}
- close(sessionState);
+ close(sessionState, keepTmpDir);
sessionState.open(conf, additionalFiles);
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java?rev=1628512&r1=1628511&r2=1628512&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java Tue Sep 30 18:36:56 2014
@@ -370,7 +370,7 @@ public class TezTask extends Task<TezWor
console.printInfo("Tez session was closed. Reopening...");
// close the old one, but keep the tmp files around
- TezSessionPoolManager.getInstance().closeAndOpen(sessionState, this.conf, inputOutputJars);
+ TezSessionPoolManager.getInstance().closeAndOpen(sessionState, this.conf, inputOutputJars, true);
console.printInfo("Session re-established.");
dagClient = sessionState.getSession().submitDAG(dag);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1628512&r1=1628511&r2=1628512&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Tue Sep 30 18:36:56 2014
@@ -1245,7 +1245,7 @@ public class SessionState {
try {
if (tezSessionState != null) {
- TezSessionPoolManager.getInstance().close(tezSessionState);
+ TezSessionPoolManager.getInstance().close(tezSessionState, false);
}
} catch (Exception e) {
LOG.info("Error closing tez session", e);
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java?rev=1628512&r1=1628511&r2=1628512&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionPool.java Tue Sep 30 18:36:56 2014
@@ -165,7 +165,7 @@ public class TestTezSessionPool {
TezSessionState session = Mockito.mock(TezSessionState.class);
Mockito.when(session.isDefault()).thenReturn(false);
- poolManager.closeAndOpen(session, conf);
+ poolManager.closeAndOpen(session, conf, false);
Mockito.verify(session).close(false);
Mockito.verify(session).open(conf, null);
@@ -178,7 +178,7 @@ public class TestTezSessionPool {
Mockito.when(session.isDefault()).thenReturn(false);
String[] extraResources = new String[] { "file:///tmp/foo.jar" };
- poolManager.closeAndOpen(session, conf, extraResources);
+ poolManager.closeAndOpen(session, conf, extraResources, false);
Mockito.verify(session).close(false);
Mockito.verify(session).open(conf, extraResources);
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java?rev=1628512&r1=1628511&r2=1628512&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java Tue Sep 30 18:36:56 2014
@@ -214,7 +214,7 @@ public class TestTezTask {
new String[0], Collections.<String,LocalResource> emptyMap());
// validate close/reopen
verify(sessionState, times(1)).open(any(HiveConf.class), any(String[].class));
- verify(sessionState, times(1)).close(eq(false)); // now uses pool after HIVE-7043
+ verify(sessionState, times(1)).close(eq(true)); // now uses pool after HIVE-7043
verify(session, times(2)).submitDAG(any(DAG.class));
}