svn commit: r1532661 - in /hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec: Utilities.java tez/DagUtils.java tez/MapRecordProcessor.java tez/ReduceRecordProcessor.java

[gu...@apache.org]

Author: gunther
Date: Wed Oct 16 07:20:50 2013
New Revision: 1532661

URL: http://svn.apache.org/r1532661
Log:
HIVE-5561: Clear work map for container reuse on tez (Gunther Hagleitner)

Modified:
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1532661&r1=1532660&r2=1532661&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Wed Oct 16 07:20:50 2013
@@ -2877,7 +2877,7 @@ public final class Utilities {
           pathsProcessed.add(path);
 
           LOG.info("Adding input file " + path);
-          if (isEmptyPath(job, path, ctx)) {
+          if (!HiveConf.getBoolVar(job, ConfVars.HIVE_OPTIMIZE_TEZ) && isEmptyPath(job, path, ctx)) {
             path = createDummyFileForEmptyPartition(path, job, work,
                  hiveScratchDir, alias, sequenceNumber++);
 
@@ -2894,7 +2894,7 @@ public final class Utilities {
       // T2) x;
       // If T is empty and T2 contains 100 rows, the user expects: 0, 100 (2
       // rows)
-      if (path == null) {
+      if (path == null && !HiveConf.getBoolVar(job, ConfVars.HIVE_OPTIMIZE_TEZ)) {
         path = createDummyFileForEmptyTable(job, work, hiveScratchDir,
             alias, sequenceNumber++);
         pathsToAdd.add(path);
@@ -3174,5 +3174,9 @@ public final class Utilities {
 
     return getSize(conf, size, path);
   }
+
+  public static void clearWorkMap() {
+    gWorkMap.clear();
+  }
 }
 

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java?rev=1532661&r1=1532660&r2=1532661&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java Wed Oct 16 07:20:50 2013
@@ -522,13 +522,7 @@ public class DagUtils {
   public static JobConf createConfiguration(HiveConf hiveConf) throws IOException {
     hiveConf.setBoolean("mapred.mapper.new-api", false);
 
-    JobConf conf = (JobConf) MRHelpers.getBaseMRConfiguration();
-
-    for (Map.Entry<String, String> entry: hiveConf) {
-      if (conf.get(entry.getKey()) == null) {
-          conf.set(entry.getKey(), entry.getValue());
-      }
-    }
+    JobConf conf = (JobConf) MRHelpers.getBaseMRConfiguration(hiveConf);
 
     conf.set("mapred.output.committer.class", NullOutputCommitter.class.getName());
 

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java?rev=1532661&r1=1532660&r2=1532661&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java Wed Oct 16 07:20:50 2013
@@ -234,6 +234,7 @@ public class MapRecordProcessor  extends
         throw new RuntimeException("Hive Runtime Error while closing operators", e);
       }
     } finally {
+      Utilities.clearWorkMap();
       MapredContext.close();
     }
   }

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java?rev=1532661&r1=1532660&r2=1532661&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java Wed Oct 16 07:20:50 2013
@@ -357,6 +357,7 @@ public class ReduceRecordProcessor  exte
             + e.getMessage(), e);
       }
     } finally {
+      Utilities.clearWorkMap();
       MapredContext.close();
     }
   }