You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by pr...@apache.org on 2010/03/13 21:02:38 UTC
svn commit: r922664 - in /hadoop/pig/trunk: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java
Author: pradeepkth
Date: Sat Mar 13 20:02:38 2010
New Revision: 922664
URL: http://svn.apache.org/viewvc?rev=922664&view=rev
Log:
PIG-1290: WeightedRangePartitioner should not check if input is empty if quantile file is empty (pradeepkth)
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=922664&r1=922663&r2=922664&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Sat Mar 13 20:02:38 2010
@@ -66,6 +66,9 @@ manner (rding via pradeepkth)
IMPROVEMENTS
+PIG-1290: WeightedRangePartitioner should not check if input is empty if
+quantile file is empty (pradeepkth)
+
PIG-1262: Additional findbugs and javac warnings (daijy)
PIG-1248: [piggybank] some useful String functions (dvryaboy)
Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java?rev=922664&r1=922663&r2=922664&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/partitioners/WeightedRangePartitioner.java Sat Mar 13 20:02:38 2010
@@ -125,33 +125,12 @@ public class WeightedRangePartitioner ex
new DiscreteProbabilitySampleGenerator(probVec));
}
}
- else {
- ArrayList<FileSpec> inp =
- (ArrayList<FileSpec>)
- ObjectSerializer.deserialize(job.get("pig.inputs", ""));
- //order-by MR job will have only one input
- FileSpec fileSpec = inp.get(0);
- LoadFunc inpLoad =
- (LoadFunc)PigContext.instantiateFuncFromSpec(fileSpec.getFuncSpec());
- List<String> inpSignatureLists =
- (ArrayList<String>)ObjectSerializer.deserialize(
- job.get("pig.inpSignatures"));
- // signature can be null for intermediate jobs where it will not
- // be required to be passed down
- if(inpSignatureLists.get(0) != null) {
- inpLoad.setUDFContextSignature(inpSignatureLists.get(0));
- }
-
- ReadToEndLoader r2eLoad = new ReadToEndLoader(inpLoad, job,
- fileSpec.getFileName(), 0);
-
- if (r2eLoad.getNext() != null)
- {
- throw new RuntimeException("Empty samples file and non-empty input file");
- }
- // Otherwise, we do not put anything to weightedParts
- }
-
+ // else - the quantiles file is empty - unless we have a bug, the
+ // input must also be empty in which case we don't need to put
+ // anything in weightedParts since getPartition() should never get
+ // called. If the quantiles file is empty due to either a bug or
+ // a transient failure situation on the dfs, then weightedParts will
+ // not be populated and the job will fail in getPartition()
}catch (Exception e){
throw new RuntimeException(e);
}