You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ha...@apache.org on 2010/02/04 22:29:16 UTC
svn commit: r906657 - in /hadoop/pig/trunk: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POJoinPackage.java
Author: hashutosh
Date: Thu Feb 4 21:29:16 2010
New Revision: 906657
URL: http://svn.apache.org/viewvc?rev=906657&view=rev
Log:
PIG-1209: Port POJoinPackage to proactively spill
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POJoinPackage.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=906657&r1=906656&r2=906657&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Thu Feb 4 21:29:16 2010
@@ -24,6 +24,8 @@
IMPROVEMENTS
+PIG-1209: Port POJoinPackage to proactively spill (ashutoshc)
+
PIG-1190: Handling of quoted strings in pig-latin/grunt commands (ashutoshc)
PIG-1214: Pig 0.6 Docs fixes (chandec via olgan)
Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POJoinPackage.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POJoinPackage.java?rev=906657&r1=906656&r2=906657&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POJoinPackage.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POJoinPackage.java Thu Feb 4 21:29:16 2010
@@ -20,12 +20,14 @@
import java.util.List;
import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataType;
+import org.apache.pig.data.InternalCachedBag;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.io.NullableTuple;
import org.apache.pig.impl.plan.NodeIdGenerator;
@@ -124,9 +126,17 @@
{
lastInputTuple = false;
//Put n-1 inputs into bags
+ String bagType = null;
+ if (PigMapReduce.sJobConf != null) {
+ bagType = PigMapReduce.sJobConf.get("pig.cachedbag.type");
+ }
dbs = new DataBag[numInputs];
for (int i = 0; i < numInputs; i++) {
- dbs[i] = mBagFactory.newDefaultBag();
+ if (bagType != null && bagType.equalsIgnoreCase("default")) {
+ dbs[i] = mBagFactory.newDefaultBag();
+ } else {
+ dbs[i] = new InternalCachedBag(numInputs);
+ }
}
//For each Nullable tuple in the input, put it