svn commit: r1638236 - /pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java

[da...@apache.org]

Author: daijy
Date: Tue Nov 11 18:15:18 2014
New Revision: 1638236

URL: http://svn.apache.org/r1638236
Log:
PIG-4314: BigData_5 hang on some machine (PIG-4314-5.patch)

Modified:
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java?rev=1638236&r1=1638235&r2=1638236&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java Tue Nov 11 18:15:18 2014
@@ -36,9 +36,9 @@ import org.apache.pig.backend.hadoop.exe
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.ExpressionOperator;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
-import org.apache.pig.data.BagFactory;
 import org.apache.pig.data.DataBag;
 import org.apache.pig.data.DataType;
+import org.apache.pig.data.InternalCachedBag;
 import org.apache.pig.data.SelfSpillBag.MemoryLimits;
 import org.apache.pig.data.Tuple;
 import org.apache.pig.data.TupleFactory;
@@ -84,7 +84,6 @@ public class POPartialAgg extends Physic
     private static final WeakHashMap<POPartialAgg, Byte> ALL_POPARTS = new WeakHashMap<POPartialAgg, Byte>();
 
     private static final TupleFactory TF = TupleFactory.getInstance();
-    private static final BagFactory BG = BagFactory.getInstance();
 
     private PhysicalPlan keyPlan;
     private ExpressionOperator keyLeaf;
@@ -459,7 +458,14 @@ public class POPartialAgg extends Physic
         valueTuple.set(0, key);
 
         for (int i = 0; i < valuePlans.size(); i++) {
-            DataBag bag = BG.newDefaultBag();
+            DataBag bag = null;
+            if (doContingentSpill) {
+                // Don't use additional memory since we already have memory stress
+                bag = new InternalCachedBag();
+            } else {
+                // Take 10% of memory, need fine tune later
+                bag = new InternalCachedBag(1, 0.1F);
+            }
             valueTuple.set(i + 1, bag);
         }
         for (Tuple t : inpTuples) {