[tez] branch branch-0.9 updated: TEZ-4273: Clear off staging files when TezYarnClient is unable to submit applications (Rajesh Balamohan via László Bodor)

[ab...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

abstractdog pushed a commit to branch branch-0.9
in repository https://gitbox.apache.org/repos/asf/tez.git


The following commit(s) were added to refs/heads/branch-0.9 by this push:
     new f3ee736  TEZ-4273: Clear off staging files when TezYarnClient is unable to submit applications (Rajesh Balamohan via László Bodor)
f3ee736 is described below

commit f3ee736575bcbb44d07da3713e4e42e8bad0acc9
Author: Rajesh Balamohan <rb...@apache.org>
AuthorDate: Mon Feb 8 09:44:22 2021 +0100

    TEZ-4273: Clear off staging files when TezYarnClient is unable to submit applications (Rajesh Balamohan via László Bodor)
    
    Signed-off-by: Laszlo Bodor <bo...@gmail.com>
---
 .../main/java/org/apache/tez/client/TezClient.java | 24 ++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
index a615179..949a2ce 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
@@ -34,6 +34,7 @@ import java.util.Objects;
 
 import javax.annotation.Nullable;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
@@ -405,6 +406,10 @@ public class TezClient {
         LOG.info("The url to track the Tez Session: " + appReport.getTrackingUrl());
         sessionStarted.set(true);
       } catch (YarnException e) {
+        cleanStagingDir();
+        throw new TezException(e);
+      } catch (IOException e) {
+        cleanStagingDir();
         throw new TezException(e);
       }
 
@@ -413,6 +418,21 @@ public class TezClient {
     }
   }
 
+  private void cleanStagingDir() {
+    Configuration conf = amConfig.getTezConfiguration();
+    String appId = sessionAppId.toString();
+    Path stagingDir = TezCommonUtils.getTezSystemStagingPath(conf, appId);
+    boolean isStgDeleted = false;
+    try {
+      FileSystem fs = stagingDir.getFileSystem(conf);
+      isStgDeleted = fs.delete(stagingDir, true);
+    } catch (IOException ioe) {
+      LOG.error("Error deleting staging dir " + stagingDir, ioe);
+    } finally {
+      LOG.info("Staging dir {}, deleted:{} ", stagingDir, isStgDeleted);
+    }
+  }
+
   public synchronized TezClient getClient(String appIdStr) throws IOException, TezException {
     return getClient(appIdfromString(appIdStr));
   }
@@ -453,6 +473,10 @@ public class TezClient {
       LOG.info("The url to track the Tez Session: " + appReport.getTrackingUrl());
       sessionStarted.set(true);
     } catch (YarnException e) {
+      cleanStagingDir();
+      throw new TezException(e);
+    } catch (IOException e) {
+      cleanStagingDir();
       throw new TezException(e);
     }