You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2010/05/03 22:13:32 UTC
svn commit: r940595 - in /hadoop/pig/branches/branch-0.7: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POCombinerPackage.java
Author: daijy
Date: Mon May 3 20:13:31 2010
New Revision: 940595
URL: http://svn.apache.org/viewvc?rev=940595&view=rev
Log:
PIG-1394: POCombinerPackage hold too much memory for InternalCachedBag
Modified:
hadoop/pig/branches/branch-0.7/CHANGES.txt
hadoop/pig/branches/branch-0.7/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POCombinerPackage.java
Modified: hadoop/pig/branches/branch-0.7/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/CHANGES.txt?rev=940595&r1=940594&r2=940595&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.7/CHANGES.txt (original)
+++ hadoop/pig/branches/branch-0.7/CHANGES.txt Mon May 3 20:13:31 2010
@@ -187,6 +187,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-1394: POCombinerPackage hold too much memory for InternalCachedBag (daijy)
+
PIG-1303: Inconsistent instantiation of parametrized UDFs (jrussek and dvryaboy)
PIG-1348: PigStorage making unnecessary byte array copy when storing data
Modified: hadoop/pig/branches/branch-0.7/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POCombinerPackage.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POCombinerPackage.java?rev=940595&r1=940594&r2=940595&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.7/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POCombinerPackage.java (original)
+++ hadoop/pig/branches/branch-0.7/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POCombinerPackage.java Mon May 3 20:13:31 2010
@@ -66,7 +66,8 @@ public class POCombinerPackage extends P
private boolean[] keyPositions;
private Map<Integer, Integer> keyLookup;
-
+
+ private int numBags;
/**
* A new POPostCombinePackage will be constructed as a near clone of the
@@ -91,6 +92,10 @@ public class POCombinerPackage extends P
if (bags != null) {
mBags = Arrays.copyOf(bags, bags.length);
}
+ numBags = 0;
+ for (int i = 0; i < mBags.length; i++) {
+ if (mBags[i]) numBags++;
+ }
if (keyPos != null) {
keyPositions = Arrays.copyOf(keyPos, keyPos.length);
}
@@ -122,7 +127,7 @@ public class POCombinerPackage extends P
keyLookup = lrKeyInfo.second;
}
- private DataBag createDataBag() {
+ private DataBag createDataBag(int numBags) {
String bagType = null;
if (PigMapReduce.sJobConf != null) {
bagType = PigMapReduce.sJobConf.get("pig.cachedbag.type");
@@ -131,7 +136,7 @@ public class POCombinerPackage extends P
if (bagType != null && bagType.equalsIgnoreCase("default")) {
return new NonSpillableDataBag();
}
- return new InternalCachedBag();
+ return new InternalCachedBag(numBags);
}
@Override
@@ -140,7 +145,7 @@ public class POCombinerPackage extends P
//Create numInputs bags
Object[] fields = new Object[mBags.length];
for (int i = 0; i < mBags.length; i++) {
- if (mBags[i]) fields[i] = createDataBag();
+ if (mBags[i]) fields[i] = createDataBag(numBags);
}
// For each indexed tup in the inp, split them up and place their