You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2014/01/30 17:37:25 UTC
svn commit: r1562886 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/data/SelfSpillBag.java
Author: rohini
Date: Thu Jan 30 16:37:24 2014
New Revision: 1562886
URL: http://svn.apache.org/r1562886
Log:
PIG-3730: Performance issue in SelfSpillBag (rajesh.balamohan via rohini)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/data/SelfSpillBag.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1562886&r1=1562885&r2=1562886&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Jan 30 16:37:24 2014
@@ -30,6 +30,8 @@ PIG-2207: Support custom counters for ag
IMPROVEMENTS
+PIG-3730: Performance issue in SelfSpillBag (rajesh.balamohan via rohini)
+
PIG-3654: Add class cache to PigContext (tmwoodruff via daijy)
PIG-3463: Pig should use hadoop local mode for small jobs (aniket486)
@@ -76,7 +78,7 @@ BUG FIXES
PIG-3682: mvn-inst target does not install pig-h2.jar into local .m2 (raluri via aniket486)
-PIG-3677: ConfigurationUtil.getLocalFSProperties can return an inconsistent property set (rohini)
+PIG-3661: Piggybank AvroStorage fails if used in more than one load or store statement (rohini)
PIG-3511: Security: Pig temporary directories might have world readable permissions (rohini)
@@ -179,6 +181,8 @@ PIG-3480: TFile-based tmpfile compressio
BUG FIXES
+PIG-3677: ConfigurationUtil.getLocalFSProperties can return an inconsistent property set (rohini)
+
PIG-3621: Python Avro library can't read Avros made with builtin AvroStorage (rusell.jurney via cheolsoo)
PIG-3592: Should not try to create success file for non-fs schemes like hbase (rohini)
Modified: pig/trunk/src/org/apache/pig/data/SelfSpillBag.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/data/SelfSpillBag.java?rev=1562886&r1=1562885&r2=1562886&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/data/SelfSpillBag.java (original)
+++ pig/trunk/src/org/apache/pig/data/SelfSpillBag.java Thu Jan 30 16:37:24 2014
@@ -56,6 +56,19 @@ public abstract class SelfSpillBag exten
private int cacheLimit = Integer.MAX_VALUE;
private long memUsage = 0;
private long numObjsSizeChecked = 0;
+
+ private static float cachedMemUsage = 0.2F;
+ private static long maxMem = 0;
+ static {
+ maxMem = Runtime.getRuntime().maxMemory();
+ if (PigMapReduce.sJobConfInternal.get() != null) {
+ String usage = PigMapReduce.sJobConfInternal.get().get(
+ PigConfiguration.PROP_CACHEDBAG_MEMUSAGE);
+ if (usage != null) {
+ cachedMemUsage = Float.parseFloat(usage);
+ }
+ }
+ }
/**
* @param bagCount
@@ -68,18 +81,10 @@ public abstract class SelfSpillBag exten
private void init(int bagCount, float percent) {
if (percent < 0) {
- percent = 0.2F;
- if (PigMapReduce.sJobConfInternal.get() != null) {
- String usage = PigMapReduce.sJobConfInternal.get().get(
- PigConfiguration.PROP_CACHEDBAG_MEMUSAGE);
- if (usage != null) {
- percent = Float.parseFloat(usage);
- }
- }
+ percent = cachedMemUsage;
}
- long max = Runtime.getRuntime().maxMemory();
- maxMemUsage = (long) ((max * percent) / bagCount);
+ maxMemUsage = (long) ((maxMem * percent) / bagCount);
// set limit to 0, if memusage is 0 or really really small.
// then all tuples are put into disk