You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/11/11 19:15:06 UTC
svn commit: r1638235 -
/pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java
Author: daijy
Date: Tue Nov 11 18:15:06 2014
New Revision: 1638235
URL: http://svn.apache.org/r1638235
Log:
PIG-4314: BigData_5 hang on some machine (PIG-4314-5.patch)
Modified:
pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java
Modified: pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java?rev=1638235&r1=1638234&r2=1638235&view=diff
==============================================================================
--- pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java (original)
+++ pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPartialAgg.java Tue Nov 11 18:15:06 2014
@@ -36,9 +36,9 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.ExpressionOperator;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
-import org.apache.pig.data.BagFactory;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataType;
+import org.apache.pig.data.InternalCachedBag;
import org.apache.pig.data.SelfSpillBag.MemoryLimits;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
@@ -84,7 +84,6 @@ public class POPartialAgg extends Physic
private static final WeakHashMap<POPartialAgg, Byte> ALL_POPARTS = new WeakHashMap<POPartialAgg, Byte>();
private static final TupleFactory TF = TupleFactory.getInstance();
- private static final BagFactory BG = BagFactory.getInstance();
private PhysicalPlan keyPlan;
private ExpressionOperator keyLeaf;
@@ -459,7 +458,14 @@ public class POPartialAgg extends Physic
valueTuple.set(0, key);
for (int i = 0; i < valuePlans.size(); i++) {
- DataBag bag = BG.newDefaultBag();
+ DataBag bag = null;
+ if (doContingentSpill) {
+ // Don't use additional memory since we already have memory stress
+ bag = new InternalCachedBag();
+ } else {
+ // Take 10% of memory, need fine tune later
+ bag = new InternalCachedBag(1, 0.1F);
+ }
valueTuple.set(i + 1, bag);
}
for (Tuple t : inpTuples) {