svn commit: r826042 - in /hadoop/hive/trunk: CHANGES.txt ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java

[zs...@apache.org]

Author: zshao
Date: Fri Oct 16 19:08:53 2009
New Revision: 826042

URL: http://svn.apache.org/viewvc?rev=826042&view=rev
Log:
HIVE-882. Create a new directory every time for scratch. (Namit Jain via zshao)

Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=826042&r1=826041&r2=826042&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Fri Oct 16 19:08:53 2009
@@ -190,6 +190,9 @@
     HIVE-878. Update the hash table entry before flushing in Group By
     hash aggregation. (Zheng Shao via namit)
 
+    HIVE-882. Create a new directory every time for scratch.
+    (Namit Jain via zshao)
+
 Release 0.4.0 -  Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java?rev=826042&r1=826041&r2=826042&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java Fri Oct 16 19:08:53 2009
@@ -408,7 +408,28 @@
 
 
     String hiveScratchDir = HiveConf.getVar(job, HiveConf.ConfVars.SCRATCHDIR);
-    Path jobScratchDir = new Path(hiveScratchDir + Utilities.randGen.nextInt());
+    String jobScratchDirStr = hiveScratchDir + File.separator+ Utilities.randGen.nextInt();
+    Path   jobScratchDir = new Path(jobScratchDirStr);
+    String emptyScratchDirStr = null;
+    Path   emptyScratchDir    = null;
+
+    int numTries = 3;
+    while (numTries > 0) {
+      emptyScratchDirStr = hiveScratchDir + File.separator + Utilities.randGen.nextInt();
+      emptyScratchDir = new Path(emptyScratchDirStr);
+
+      try {
+        FileSystem fs = emptyScratchDir.getFileSystem(job);
+        fs.mkdirs(emptyScratchDir);
+        break;
+      } catch (Exception e) {
+        if (numTries > 0)
+          numTries--;
+        else
+          throw new RuntimeException("Failed to make dir " + emptyScratchDir.toString() + " : " + e.getMessage());
+      }
+    }
+
     FileOutputFormat.setOutputPath(job, jobScratchDir);
     job.setMapperClass(ExecMapper.class);
 
@@ -462,7 +483,7 @@
     boolean success = false;
 
     try {
-      addInputPaths(job, work, hiveScratchDir.toString());
+      addInputPaths(job, work, emptyScratchDirStr);
       Utilities.setMapRedWork(job, work);
 
       // remove the pwd from conf file so that job tracker doesn't show this logs
@@ -525,6 +546,7 @@
       try {
         FileSystem fs = jobScratchDir.getFileSystem(job);
         fs.delete(jobScratchDir, true);
+        fs.delete(emptyScratchDir, true);
         if (returnVal != 0 && rj != null) {
           rj.killJob();
         }