You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ol...@apache.org on 2007/11/17 01:21:48 UTC
svn commit: r595880 - in /incubator/pig/trunk: CHANGES.txt
src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java
Author: olga
Date: Fri Nov 16 16:21:46 2007
New Revision: 595880
URL: http://svn.apache.org/viewvc?rev=595880&view=rev
Log:
added binary comparator; PIG-8
Modified:
incubator/pig/trunk/CHANGES.txt
incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java
Modified: incubator/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/CHANGES.txt?rev=595880&r1=595879&r2=595880&view=diff
==============================================================================
--- incubator/pig/trunk/CHANGES.txt (original)
+++ incubator/pig/trunk/CHANGES.txt Fri Nov 16 16:21:46 2007
@@ -18,4 +18,4 @@
PIG-23 Made pig work with java 1.5. (milindb via gates)
-
+ PIG-8 added binary comparator (olgan)
Modified: incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java
URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java?rev=595880&r1=595879&r2=595880&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java Fri Nov 16 16:21:46 2007
@@ -40,6 +40,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.UTF8;
+import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.TaskReport;
import org.apache.hadoop.mapred.JobClient;
@@ -63,6 +64,16 @@
numMRJobs = numMRJobsIn;
mrJobNumber = 0;
}
+
+ public static class PigWritableComparator extends WritableComparator {
+ public PigWritableComparator() {
+ super(Tuple.class);
+ }
+
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2){
+ return WritableComparator.compareBytes(b1, s1, l1, b2, s2, l2);
+ }
+ }
static Random rand = new Random();
@@ -136,7 +147,11 @@
conf.setCombinerClass(PigCombine.class);
if (pom.quantilesFile!=null){
conf.set("pig.quantilesFile", pom.quantilesFile);
- }
+ }
+ else{
+ // this is not a sort job - can use byte comparison to speed up processing
+ conf.setOutputKeyComparatorClass(PigWritableComparator.class);
+ }
if (pom.partitionFunction!=null){
conf.setPartitionerClass(SortPartitioner.class);
}