You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2018/01/10 16:25:29 UTC

hive git commit: HIVE-18417 : better error handling in TezSessionState cleanup (Sergey Shelukhin via Jason Dere)

Repository: hive
Updated Branches:
  refs/heads/master d69601c24 -> 19e6a8b56


HIVE-18417 : better error handling in TezSessionState cleanup (Sergey Shelukhin via Jason Dere)

Signed-off-by: Ashutosh Chauhan <ha...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/19e6a8b5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/19e6a8b5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/19e6a8b5

Branch: refs/heads/master
Commit: 19e6a8b560c67313ff358e2c183aa47b443fc033
Parents: d69601c
Author: Sergey Shelukhin <se...@apache.org>
Authored: Wed Jan 10 08:24:19 2018 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Wed Jan 10 08:24:19 2018 -0800

----------------------------------------------------------------------
 .../hive/ql/exec/tez/TezSessionState.java       | 89 ++++++++++++--------
 .../apache/hadoop/hive/ql/exec/tez/TezTask.java |  5 +-
 2 files changed, 59 insertions(+), 35 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/19e6a8b5/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
index d66cf11..ebb278d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
@@ -37,7 +37,9 @@ import java.util.concurrent.FutureTask;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicReference;
+
 import javax.security.auth.login.LoginException;
+
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.hadoop.conf.Configuration;
@@ -90,6 +92,7 @@ import org.codehaus.jackson.map.annotate.JsonSerialize;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.hive.ql.exec.tez.monitoring.TezJobMonitor;
+
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
 
@@ -134,6 +137,12 @@ public class TezSessionState {
     public final Set<String> additionalFilesNotFromConf = new HashSet<>();
     /** Localized resources of this session; both from conf and not from conf (above). */
     public final Set<LocalResource> localizedResources = new HashSet<>();
+
+    @Override
+    public String toString() {
+      return dagResourcesDir + "; " + additionalFilesNotFromConf.size() + " additional files, "
+          + localizedResources.size() + " localized resources";
+    }
   }
 
   private HiveResources resources;
@@ -259,9 +268,11 @@ public class TezSessionState {
     if (resources != null) {
       // If we are getting the resources externally, don't relocalize anything.
       this.resources = resources;
+      LOG.info("Setting resources to " + resources);
     } else {
       this.resources = new HiveResources(createTezDir(sessionId, "resources"));
       ensureLocalResources(conf, additionalFilesNotFromConf);
+      LOG.info("Created new resources: " + resources);
     }
 
     // unless already installed on all the cluster nodes, we'll have to
@@ -555,6 +566,9 @@ public class TezSessionState {
   /** This is called in openInternal and in TezTask.updateSession to localize conf resources. */
   public void ensureLocalResources(Configuration conf, String[] newFilesNotFromConf)
           throws IOException, LoginException, URISyntaxException, TezException {
+    if (resources == null) {
+      throw new AssertionError("Ensure called on an unitialized (or closed) session " + sessionId);
+    }
     String dir = resources.dagResourcesDir.toString();
     resources.localizedResources.clear();
 
@@ -614,36 +628,40 @@ public class TezSessionState {
    * @throws Exception
    */
   void close(boolean keepDagFilesDir) throws Exception {
-    if (session != null) {
-      LOG.info("Closing Tez Session");
-      closeClient(session);
-    } else if (sessionFuture != null) {
-      sessionFuture.cancel(true);
-      TezClient asyncSession = null;
-      try {
-        asyncSession = sessionFuture.get(); // In case it was done and noone looked at it.
-      } catch (ExecutionException | CancellationException e) {
-        // ignore
-      } catch (InterruptedException e) {
-        Thread.currentThread().interrupt();
-        // ignore
-      }
-      if (asyncSession != null) {
+    console = null;
+    appJarLr = null;
+
+    try {
+      if (session != null) {
         LOG.info("Closing Tez Session");
-        closeClient(asyncSession);
+        closeClient(session);
+        session = null;
+      } else if (sessionFuture != null) {
+        sessionFuture.cancel(true);
+        TezClient asyncSession = null;
+        try {
+          asyncSession = sessionFuture.get(); // In case it was done and noone looked at it.
+        } catch (ExecutionException | CancellationException e) {
+          // ignore
+        } catch (InterruptedException e) {
+          Thread.currentThread().interrupt();
+          // ignore
+        }
+        sessionFuture = null;
+        if (asyncSession != null) {
+          LOG.info("Closing Tez Session");
+          closeClient(asyncSession);
+        }
+      }
+    } finally {
+      try {
+        cleanupScratchDir();
+      } finally {
+        if (!keepDagFilesDir) {
+          cleanupDagResources();
+        }
       }
     }
-
-    cleanupScratchDir();
-    if (!keepDagFilesDir) {
-      cleanupDagResources();
-    }
-    session = null;
-    sessionFuture = null;
-    console = null;
-    tezScratchDir = null;
-    // Do not reset dag resources; if it wasn't cleaned it's still needed.
-    appJarLr = null;
   }
 
   private void closeClient(TezClient client) throws TezException,
@@ -656,15 +674,20 @@ public class TezSessionState {
   }
 
   protected final void cleanupScratchDir() throws IOException {
-    FileSystem fs = tezScratchDir.getFileSystem(conf);
-    fs.delete(tezScratchDir, true);
-    tezScratchDir = null;
+    if (tezScratchDir != null) {
+      FileSystem fs = tezScratchDir.getFileSystem(conf);
+      fs.delete(tezScratchDir, true);
+      tezScratchDir = null;
+    }
   }
 
   protected final void cleanupDagResources() throws IOException {
-    FileSystem fs = resources.dagResourcesDir.getFileSystem(conf);
-    fs.delete(resources.dagResourcesDir, true);
-    resources = null;
+    LOG.info("Attemting to clean up resources for " + sessionId + ": " + resources);
+    if (resources != null) {
+      FileSystem fs = resources.dagResourcesDir.getFileSystem(conf);
+      fs.delete(resources.dagResourcesDir, true);
+      resources = null;
+    }
   }
 
   public String getSessionId() {

http://git-wip-us.apache.org/repos/asf/hive/blob/19e6a8b5/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
index ab3a71e..2756bf7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
@@ -516,7 +516,6 @@ public class TezTask extends Task<TezWork> {
   }
 
   DAGClient submit(JobConf conf, DAG dag, Ref<TezSessionState> sessionStateRef) throws Exception {
-
     perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_SUBMIT_DAG);
     DAGClient dagClient = null;
     TezSessionState sessionState = sessionStateRef.value;
@@ -526,6 +525,7 @@ public class TezTask extends Task<TezWork> {
         dagClient = sessionState.getSession().submitDAG(dag);
       } catch (SessionNotRunning nr) {
         console.printInfo("Tez session was closed. Reopening...");
+        sessionStateRef.value = null;
         sessionStateRef.value = sessionState = getNewTezSessionOnError(sessionState);
         console.printInfo("Session re-established.");
         dagClient = sessionState.getSession().submitDAG(dag);
@@ -535,12 +535,13 @@ public class TezTask extends Task<TezWork> {
       try {
         console.printInfo("Dag submit failed due to " + e.getMessage() + " stack trace: "
             + Arrays.toString(e.getStackTrace()) + " retrying...");
+        sessionStateRef.value = null;
         sessionStateRef.value = sessionState = getNewTezSessionOnError(sessionState);
         dagClient = sessionState.getSession().submitDAG(dag);
       } catch (Exception retryException) {
         // we failed to submit after retrying. Destroy session and bail.
-        sessionState.destroy();
         sessionStateRef.value = null;
+        sessionState.destroy();
         throw retryException;
       }
     }