You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@chukwa.apache.org by as...@apache.org on 2012/07/17 02:06:37 UTC
svn commit: r1362318 - in /incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction: CHUKWA_CONSTANT.java demux/ChukwaRecordPartitioner.java

Author: asrabkin
Date: Tue Jul 17 00:06:37 2012
New Revision: 1362318

URL: http://svn.apache.org/viewvc?rev=1362318&view=rev
Log:
CHUKWA-647. Spread out intermediate data with the same ReduceType into different Reduce Tasks. Contributed by Jie Huang.

Modified:
    incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/CHUKWA_CONSTANT.java
    incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/ChukwaRecordPartitioner.java

Modified: incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/CHUKWA_CONSTANT.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/CHUKWA_CONSTANT.java?rev=1362318&r1=1362317&r2=1362318&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/CHUKWA_CONSTANT.java (original)
+++ incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/CHUKWA_CONSTANT.java Tue Jul 17 00:06:37 2012
@@ -64,5 +64,6 @@ public interface CHUKWA_CONSTANT {
   public static final String ARCHIVES_IN_ERROR_DIR_NAME      = "inError/";
 
   public static final String POST_DEMUX_DATA_LOADER = "chukwa.post.demux.data.loader";  
+  public static final String INCLUDE_KEY_IN_PARTITIONER = "_";
   public static final String POST_DEMUX_SUCCESS_ACTION = "chukwa.post.demux.success.action";  
 }

Modified: incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/ChukwaRecordPartitioner.java
URL: http://svn.apache.org/viewvc/incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/ChukwaRecordPartitioner.java?rev=1362318&r1=1362317&r2=1362318&view=diff
==============================================================================
--- incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/ChukwaRecordPartitioner.java (original)
+++ incubator/chukwa/trunk/src/main/java/org/apache/hadoop/chukwa/extraction/demux/ChukwaRecordPartitioner.java Tue Jul 17 00:06:37 2012
@@ -21,6 +21,8 @@ package org.apache.hadoop.chukwa.extract
 
 import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecord;
 import org.apache.hadoop.chukwa.extraction.engine.ChukwaRecordKey;
+import org.apache.hadoop.chukwa.extraction.CHUKWA_CONSTANT;
+
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.Partitioner;
 import org.apache.log4j.Logger;
@@ -44,7 +46,9 @@ public class ChukwaRecordPartitioner<K, 
               + "] - Reducer:"
               + ((key.getReduceType().hashCode() & Integer.MAX_VALUE) % numReduceTasks));
     }
-    return (key.getReduceType().hashCode() & Integer.MAX_VALUE)
+    String hashkey = key.getReduceType();
+    if(key.getKey().startsWith(CHUKWA_CONSTANT.INCLUDE_KEY_IN_PARTITIONER)) hashkey = key.getReduceType()+"#"+key.getKey();
+    return (hashkey.hashCode() & Integer.MAX_VALUE)
         % numReduceTasks;
   }