You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2018/01/10 16:25:29 UTC
hive git commit: HIVE-18417 : better error handling in
TezSessionState cleanup (Sergey Shelukhin via Jason Dere)
Repository: hive
Updated Branches:
refs/heads/master d69601c24 -> 19e6a8b56
HIVE-18417 : better error handling in TezSessionState cleanup (Sergey Shelukhin via Jason Dere)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/19e6a8b5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/19e6a8b5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/19e6a8b5
Branch: refs/heads/master
Commit: 19e6a8b560c67313ff358e2c183aa47b443fc033
Parents: d69601c
Author: Sergey Shelukhin <se...@apache.org>
Authored: Wed Jan 10 08:24:19 2018 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Wed Jan 10 08:24:19 2018 -0800
----------------------------------------------------------------------
.../hive/ql/exec/tez/TezSessionState.java | 89 ++++++++++++--------
.../apache/hadoop/hive/ql/exec/tez/TezTask.java | 5 +-
2 files changed, 59 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/19e6a8b5/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
index d66cf11..ebb278d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
@@ -37,7 +37,9 @@ import java.util.concurrent.FutureTask;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;
+
import javax.security.auth.login.LoginException;
+
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.hadoop.conf.Configuration;
@@ -90,6 +92,7 @@ import org.codehaus.jackson.map.annotate.JsonSerialize;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.ql.exec.tez.monitoring.TezJobMonitor;
+
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
@@ -134,6 +137,12 @@ public class TezSessionState {
public final Set<String> additionalFilesNotFromConf = new HashSet<>();
/** Localized resources of this session; both from conf and not from conf (above). */
public final Set<LocalResource> localizedResources = new HashSet<>();
+
+ @Override
+ public String toString() {
+ return dagResourcesDir + "; " + additionalFilesNotFromConf.size() + " additional files, "
+ + localizedResources.size() + " localized resources";
+ }
}
private HiveResources resources;
@@ -259,9 +268,11 @@ public class TezSessionState {
if (resources != null) {
// If we are getting the resources externally, don't relocalize anything.
this.resources = resources;
+ LOG.info("Setting resources to " + resources);
} else {
this.resources = new HiveResources(createTezDir(sessionId, "resources"));
ensureLocalResources(conf, additionalFilesNotFromConf);
+ LOG.info("Created new resources: " + resources);
}
// unless already installed on all the cluster nodes, we'll have to
@@ -555,6 +566,9 @@ public class TezSessionState {
/** This is called in openInternal and in TezTask.updateSession to localize conf resources. */
public void ensureLocalResources(Configuration conf, String[] newFilesNotFromConf)
throws IOException, LoginException, URISyntaxException, TezException {
+ if (resources == null) {
+ throw new AssertionError("Ensure called on an unitialized (or closed) session " + sessionId);
+ }
String dir = resources.dagResourcesDir.toString();
resources.localizedResources.clear();
@@ -614,36 +628,40 @@ public class TezSessionState {
* @throws Exception
*/
void close(boolean keepDagFilesDir) throws Exception {
- if (session != null) {
- LOG.info("Closing Tez Session");
- closeClient(session);
- } else if (sessionFuture != null) {
- sessionFuture.cancel(true);
- TezClient asyncSession = null;
- try {
- asyncSession = sessionFuture.get(); // In case it was done and noone looked at it.
- } catch (ExecutionException | CancellationException e) {
- // ignore
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- // ignore
- }
- if (asyncSession != null) {
+ console = null;
+ appJarLr = null;
+
+ try {
+ if (session != null) {
LOG.info("Closing Tez Session");
- closeClient(asyncSession);
+ closeClient(session);
+ session = null;
+ } else if (sessionFuture != null) {
+ sessionFuture.cancel(true);
+ TezClient asyncSession = null;
+ try {
+ asyncSession = sessionFuture.get(); // In case it was done and noone looked at it.
+ } catch (ExecutionException | CancellationException e) {
+ // ignore
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ // ignore
+ }
+ sessionFuture = null;
+ if (asyncSession != null) {
+ LOG.info("Closing Tez Session");
+ closeClient(asyncSession);
+ }
+ }
+ } finally {
+ try {
+ cleanupScratchDir();
+ } finally {
+ if (!keepDagFilesDir) {
+ cleanupDagResources();
+ }
}
}
-
- cleanupScratchDir();
- if (!keepDagFilesDir) {
- cleanupDagResources();
- }
- session = null;
- sessionFuture = null;
- console = null;
- tezScratchDir = null;
- // Do not reset dag resources; if it wasn't cleaned it's still needed.
- appJarLr = null;
}
private void closeClient(TezClient client) throws TezException,
@@ -656,15 +674,20 @@ public class TezSessionState {
}
protected final void cleanupScratchDir() throws IOException {
- FileSystem fs = tezScratchDir.getFileSystem(conf);
- fs.delete(tezScratchDir, true);
- tezScratchDir = null;
+ if (tezScratchDir != null) {
+ FileSystem fs = tezScratchDir.getFileSystem(conf);
+ fs.delete(tezScratchDir, true);
+ tezScratchDir = null;
+ }
}
protected final void cleanupDagResources() throws IOException {
- FileSystem fs = resources.dagResourcesDir.getFileSystem(conf);
- fs.delete(resources.dagResourcesDir, true);
- resources = null;
+ LOG.info("Attemting to clean up resources for " + sessionId + ": " + resources);
+ if (resources != null) {
+ FileSystem fs = resources.dagResourcesDir.getFileSystem(conf);
+ fs.delete(resources.dagResourcesDir, true);
+ resources = null;
+ }
}
public String getSessionId() {
http://git-wip-us.apache.org/repos/asf/hive/blob/19e6a8b5/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
index ab3a71e..2756bf7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
@@ -516,7 +516,6 @@ public class TezTask extends Task<TezWork> {
}
DAGClient submit(JobConf conf, DAG dag, Ref<TezSessionState> sessionStateRef) throws Exception {
-
perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_SUBMIT_DAG);
DAGClient dagClient = null;
TezSessionState sessionState = sessionStateRef.value;
@@ -526,6 +525,7 @@ public class TezTask extends Task<TezWork> {
dagClient = sessionState.getSession().submitDAG(dag);
} catch (SessionNotRunning nr) {
console.printInfo("Tez session was closed. Reopening...");
+ sessionStateRef.value = null;
sessionStateRef.value = sessionState = getNewTezSessionOnError(sessionState);
console.printInfo("Session re-established.");
dagClient = sessionState.getSession().submitDAG(dag);
@@ -535,12 +535,13 @@ public class TezTask extends Task<TezWork> {
try {
console.printInfo("Dag submit failed due to " + e.getMessage() + " stack trace: "
+ Arrays.toString(e.getStackTrace()) + " retrying...");
+ sessionStateRef.value = null;
sessionStateRef.value = sessionState = getNewTezSessionOnError(sessionState);
dagClient = sessionState.getSession().submitDAG(dag);
} catch (Exception retryException) {
// we failed to submit after retrying. Destroy session and bail.
- sessionState.destroy();
sessionStateRef.value = null;
+ sessionState.destroy();
throw retryException;
}
}