You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2013/08/13 07:35:08 UTC
svn commit: r1513351 - in /hive/trunk: ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ shims/src/0.20/java/org/apache/hadoop/hive/shims/ shims/src/0.20S/java/org/apache/hadoop/hive/shims/ shims/src/0.23/java/org/apache/hadoop/hive/shims/ shims/src/commo...

Author: gunther
Date: Tue Aug 13 05:35:07 2013
New Revision: 1513351

URL: http://svn.apache.org/r1513351
Log:
HIVE-4863: Fix parallel order by on hadoop2 (Edward Capriolo via Gunther Hagleitner)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
    hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
    hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
    hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
    hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java?rev=1513351&r1=1513350&r2=1513351&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java Tue Aug 13 05:35:07 2013
@@ -83,7 +83,6 @@ import org.apache.hadoop.mapred.JobClien
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.Partitioner;
 import org.apache.hadoop.mapred.RunningJob;
-import org.apache.hadoop.mapred.lib.TotalOrderPartitioner;
 import org.apache.log4j.Appender;
 import org.apache.log4j.BasicConfigurator;
 import org.apache.log4j.FileAppender;
@@ -505,7 +504,7 @@ public class ExecDriver extends Task<Map
     String tmpPath = context.getCtx().getExternalTmpFileURI(onePath.toUri());
 
     Path partitionFile = new Path(tmpPath, ".partitions");
-    TotalOrderPartitioner.setPartitionFile(job, partitionFile);
+    ShimLoader.getHadoopShims().setTotalOrderPartitionFile(job, partitionFile);
 
     PartitionKeySampler sampler = new PartitionKeySampler();
 

Modified: hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1513351&r1=1513350&r2=1513351&view=diff
==============================================================================
--- hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (original)
+++ hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java Tue Aug 13 05:35:07 2013
@@ -63,6 +63,7 @@ import org.apache.hadoop.mapred.TaskID;
 import org.apache.hadoop.mapred.TaskLogServlet;
 import org.apache.hadoop.mapred.lib.CombineFileInputFormat;
 import org.apache.hadoop.mapred.lib.CombineFileSplit;
+import org.apache.hadoop.mapred.lib.TotalOrderPartitioner;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.apache.hadoop.security.SecurityUtil;
@@ -77,6 +78,7 @@ import org.apache.hadoop.util.VersionInf
  * Implemention of shims against Hadoop 0.20.0.
  */
 public class Hadoop20Shims implements HadoopShims {
+
   public boolean usesJobShell() {
     return false;
   }
@@ -194,6 +196,10 @@ public class Hadoop20Shims implements Ha
     };
   }
 
+  public void setTotalOrderPartitionFile(JobConf jobConf, Path partitionFile){
+    TotalOrderPartitioner.setPartitionFile(jobConf, partitionFile);
+  }
+
   public static class InputSplitShim extends CombineFileSplit implements HadoopShims.InputSplitShim {
     long shrinkedLength;
     boolean _isShrinked;

Modified: hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1513351&r1=1513350&r2=1513351&view=diff
==============================================================================
--- hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (original)
+++ hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java Tue Aug 13 05:35:07 2013
@@ -34,6 +34,8 @@ import org.apache.hadoop.mapred.TaskLogS
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.mapred.lib.TotalOrderPartitioner;
+
 
 /**
  * Implemention of shims against Hadoop 0.20 with Security.
@@ -122,6 +124,11 @@ public class Hadoop20SShims extends Hado
     return fs.getDefaultReplication();
   }
 
+  @Override
+  public void setTotalOrderPartitionFile(JobConf jobConf, Path partitionFile){
+    TotalOrderPartitioner.setPartitionFile(jobConf, partitionFile);
+  }
+
   /**
    * Returns a shim to wrap MiniMrCluster
    */

Modified: hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1513351&r1=1513350&r2=1513351&view=diff
==============================================================================
--- hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original)
+++ hive/trunk/shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Tue Aug 13 05:35:07 2013
@@ -40,6 +40,8 @@ import org.apache.hadoop.mapreduce.task.
 import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
 import org.apache.hadoop.mapreduce.util.HostUtil;
 import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.mapred.lib.TotalOrderPartitioner;
+
 
 /**
  * Implemention of shims against Hadoop 0.23.0.
@@ -137,6 +139,11 @@ public class Hadoop23Shims extends Hadoo
     return Trash.moveToAppropriateTrash(fs, path, conf);
   }
 
+  @Override
+  public void setTotalOrderPartitionFile(JobConf jobConf, Path partitionFile){
+    TotalOrderPartitioner.setPartitionFile(jobConf, partitionFile);
+  }
+
   /**
    * Returns a shim to wrap MiniMrCluster
    */

Modified: hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1513351&r1=1513350&r2=1513351&view=diff
==============================================================================
--- hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java (original)
+++ hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java Tue Aug 13 05:35:07 2013
@@ -394,6 +394,13 @@ public interface HadoopShims {
   UserGroupInformation createProxyUser(String userName) throws IOException;
 
   /**
+   * The method sets to set the partition file has a different signature between
+   * hadoop versions.
+   * @param jobConf
+   * @param partition
+   */
+  void setTotalOrderPartitionFile(JobConf jobConf, Path partition);
+  /**
    * InputSplitShim.
    *
    */