You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2015/11/13 20:44:16 UTC
svn commit: r1714257 - in /pig/trunk: ./ src/org/apache/pig/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/
src/org/apache/pig/backend/hadoop/executionengine/tez/runtim...
Author: rohini
Date: Fri Nov 13 19:44:16 2015
New Revision: 1714257
URL: http://svn.apache.org/viewvc?rev=1714257&view=rev
Log:
PIG-4733: Avoid NullPointerException in JVMReuseImpl for builtin classes (rohini)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/JVMReuseImpl.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java
pig/trunk/src/org/apache/pig/builtin/Distinct.java
pig/trunk/src/org/apache/pig/impl/PigContext.java
pig/trunk/src/org/apache/pig/impl/util/SpillableMemoryManager.java
pig/trunk/src/org/apache/pig/impl/util/UDFContext.java
pig/trunk/src/org/apache/pig/tools/pigstats/PigStatusReporter.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1714257&r1=1714256&r2=1714257&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Nov 13 19:44:16 2015
@@ -69,6 +69,8 @@ PIG-4639: Add better parser for Apache H
BUG FIXES
+PIG-4733: Avoid NullPointerException in JVMReuseImpl for builtin classes (rohini)
+
PIG-4722: [Pig on Tez] NPE while running Combiner (rohini)
PIG-4730: [Pig on Tez] Total parallelism estimation does not account load parallelism (rohini)
Modified: pig/trunk/src/org/apache/pig/JVMReuseImpl.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/JVMReuseImpl.java?rev=1714257&r1=1714256&r2=1714257&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/JVMReuseImpl.java (original)
+++ pig/trunk/src/org/apache/pig/JVMReuseImpl.java Fri Nov 13 19:44:16 2015
@@ -22,12 +22,34 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigCombiner;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapReduce;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
+import org.apache.pig.classification.InterfaceAudience;
+import org.apache.pig.classification.InterfaceStability;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.util.SpillableMemoryManager;
+import org.apache.pig.impl.util.UDFContext;
+import org.apache.pig.tools.pigstats.PigStatusReporter;
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
public class JVMReuseImpl {
private static Log LOG = LogFactory.getLog(JVMReuseImpl.class);
public void cleanupStaticData() {
+
+ // Calling Pig builtin ones directly without reflection for optimization
+ // and to reduce probability of NPE in PIG-4418
+ SpillableMemoryManager.staticDataCleanup();
+ PhysicalOperator.staticDataCleanup();
+ PigContext.staticDataCleanup();
+ UDFContext.staticDataCleanup();
+ PigGenericMapReduce.staticDataCleanup();
+ PigStatusReporter.staticDataCleanup();
+ PigCombiner.Combine.staticDataCleanup();
+
String className = null;
String msg = null;
List<Method> staticCleanupMethods = JVMReuseManager.getInstance()
@@ -45,15 +67,16 @@ public class JVMReuseImpl {
m.invoke(null);
msg = null;
} catch (Exception e) {
- LOG.error("Exception while calling static methods:" + getMethodNames() + ". " + msg, e);
+ LOG.error("Exception while calling static methods:"
+ + getMethodNames(staticCleanupMethods) + ". " + msg, e);
throw new RuntimeException("Error while " + msg, e);
}
}
}
- private String getMethodNames() {
+ private String getMethodNames(List<Method> staticCleanupMethods) {
StringBuilder sb = new StringBuilder();
- for (Method m : JVMReuseManager.getInstance().getStaticDataCleanupMethods()) {
+ for (Method m : staticCleanupMethods) {
if (m == null) {
sb.append("null,");
} else {
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java?rev=1714257&r1=1714256&r2=1714257&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java Fri Nov 13 19:44:16 2015
@@ -27,9 +27,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.log4j.PropertyConfigurator;
-import org.apache.pig.JVMReuseManager;
import org.apache.pig.PigException;
-import org.apache.pig.StaticDataCleanup;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.HDataType;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
@@ -76,11 +74,7 @@ public class PigCombiner {
PigContext pigContext = null;
private volatile boolean initialized = false;
- static {
- JVMReuseManager.getInstance().registerForStaticDataCleanup(Combine.class);
- }
-
- @StaticDataCleanup
+ //@StaticDataCleanup
public static void staticDataCleanup() {
firstTime = true;
}
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java?rev=1714257&r1=1714256&r2=1714257&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java Fri Nov 13 19:44:16 2015
@@ -30,10 +30,8 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.jobcontrol.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.pig.JVMReuseManager;
import org.apache.pig.PigConstants;
import org.apache.pig.PigException;
-import org.apache.pig.StaticDataCleanup;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.HDataType;
import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
@@ -103,11 +101,7 @@ public class PigGenericMapReduce {
public static ThreadLocal<Configuration> sJobConfInternal = new ThreadLocal<Configuration>();
- static {
- JVMReuseManager.getInstance().registerForStaticDataCleanup(PigGenericMapReduce.class);
- }
-
- @StaticDataCleanup
+ //@StaticDataCleanup
public static void staticDataCleanup() {
sJobContext = null;
sJobConf = null;
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java?rev=1714257&r1=1714256&r2=1714257&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java Fri Nov 13 19:44:16 2015
@@ -24,8 +24,6 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.pig.JVMReuseManager;
-import org.apache.pig.StaticDataCleanup;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhyPlanVisitor;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
@@ -122,10 +120,6 @@ public abstract class PhysicalOperator e
private List<OriginalLocation> originalLocations = new ArrayList<OriginalLocation>();
- static {
- JVMReuseManager.getInstance().registerForStaticDataCleanup(PhysicalOperator.class);
- }
-
public PhysicalOperator(OperatorKey k) {
this(k, -1, null);
}
@@ -458,7 +452,7 @@ public abstract class PhysicalOperator e
PhysicalOperator.reporter.set(reporter);
}
- @StaticDataCleanup
+ //@StaticDataCleanup
public static void staticDataCleanup() {
reporter = new ThreadLocal<PigProgressable>();
}
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java?rev=1714257&r1=1714256&r2=1714257&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java Fri Nov 13 19:44:16 2015
@@ -162,7 +162,6 @@ public class PigProcessor extends Abstra
@Override
public void close() throws Exception {
-
execPlan = null;
fileOutputs = null;
leaf = null;
Modified: pig/trunk/src/org/apache/pig/builtin/Distinct.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/builtin/Distinct.java?rev=1714257&r1=1714256&r2=1714257&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/builtin/Distinct.java (original)
+++ pig/trunk/src/org/apache/pig/builtin/Distinct.java Fri Nov 13 19:44:16 2015
@@ -22,9 +22,7 @@ import java.io.IOException;
import org.apache.pig.Algebraic;
import org.apache.pig.EvalFunc;
-import org.apache.pig.JVMReuseManager;
import org.apache.pig.PigConfiguration;
-import org.apache.pig.StaticDataCleanup;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce;
import org.apache.pig.data.BagFactory;
@@ -45,16 +43,6 @@ public class Distinct extends EvalFunc<
private static boolean initialized = false;
private static boolean useDefaultBag = false;
- static {
- JVMReuseManager.getInstance().registerForStaticDataCleanup(Distinct.class);
- }
-
- @StaticDataCleanup
- public static void staticDataCleanup() {
- initialized = false;
- useDefaultBag = false;
- }
-
@Override
public DataBag exec(Tuple input) throws IOException {
return getDistinct(input);
Modified: pig/trunk/src/org/apache/pig/impl/PigContext.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/PigContext.java?rev=1714257&r1=1714256&r2=1714257&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/PigContext.java (original)
+++ pig/trunk/src/org/apache/pig/impl/PigContext.java Fri Nov 13 19:44:16 2015
@@ -52,9 +52,7 @@ import org.apache.log4j.Level;
import org.apache.pig.ExecType;
import org.apache.pig.ExecTypeProvider;
import org.apache.pig.FuncSpec;
-import org.apache.pig.JVMReuseManager;
import org.apache.pig.PigException;
-import org.apache.pig.StaticDataCleanup;
import org.apache.pig.backend.datastorage.DataStorage;
import org.apache.pig.backend.datastorage.DataStorageException;
import org.apache.pig.backend.datastorage.ElementDescriptor;
@@ -171,11 +169,7 @@ public class PigContext implements Seria
// List of paths skipped for automatic shipping
List<String> skippedShipPaths = new ArrayList<String>();
- static {
- JVMReuseManager.getInstance().registerForStaticDataCleanup(PigContext.class);
- }
-
- @StaticDataCleanup
+ //@StaticDataCleanup
public static void staticDataCleanup() {
packageImportList.set(null);
}
Modified: pig/trunk/src/org/apache/pig/impl/util/SpillableMemoryManager.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/SpillableMemoryManager.java?rev=1714257&r1=1714256&r2=1714257&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/util/SpillableMemoryManager.java (original)
+++ pig/trunk/src/org/apache/pig/impl/util/SpillableMemoryManager.java Fri Nov 13 19:44:16 2015
@@ -36,8 +36,6 @@ import javax.management.openmbean.Compos
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.pig.JVMReuseManager;
-import org.apache.pig.StaticDataCleanup;
/**
* This class Tracks the tenured pool and a list of Spillable objects. When memory gets low, this
@@ -93,12 +91,8 @@ public class SpillableMemoryManager impl
private static final SpillableMemoryManager manager = new SpillableMemoryManager();
- static {
- JVMReuseManager.getInstance().registerForStaticDataCleanup(SpillableMemoryManager.class);
- }
-
- @StaticDataCleanup
- public static void cleanupStaticData() {
+ //@StaticDataCleanup
+ public static void staticDataCleanup() {
manager.spillables.clear();
manager.accumulatedFreeSize = 0L;
}
Modified: pig/trunk/src/org/apache/pig/impl/util/UDFContext.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/UDFContext.java?rev=1714257&r1=1714256&r2=1714257&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/util/UDFContext.java (original)
+++ pig/trunk/src/org/apache/pig/impl/util/UDFContext.java Fri Nov 13 19:44:16 2015
@@ -26,8 +26,6 @@ import java.util.Map.Entry;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
-import org.apache.pig.JVMReuseManager;
-import org.apache.pig.StaticDataCleanup;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRConfiguration;
public class UDFContext {
@@ -46,10 +44,6 @@ public class UDFContext {
}
};
- static {
- JVMReuseManager.getInstance().registerForStaticDataCleanup(UDFContext.class);
- }
-
private UDFContext() {
udfConfs = new HashMap<UDFContextKey, Properties>();
}
@@ -71,8 +65,8 @@ public class UDFContext {
/*
* internal pig use only - should NOT be called from user code
*/
- @StaticDataCleanup
- public static void cleanupStaticData() {
+ //@StaticDataCleanup
+ public static void staticDataCleanup() {
tss = new ThreadLocal<UDFContext>() {
@Override
public UDFContext initialValue() {
Modified: pig/trunk/src/org/apache/pig/tools/pigstats/PigStatusReporter.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/PigStatusReporter.java?rev=1714257&r1=1714256&r2=1714257&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/PigStatusReporter.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/PigStatusReporter.java Fri Nov 13 19:44:16 2015
@@ -21,8 +21,6 @@ package org.apache.pig.tools.pigstats;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.StatusReporter;
import org.apache.hadoop.util.Progressable;
-import org.apache.pig.JVMReuseManager;
-import org.apache.pig.StaticDataCleanup;
import org.apache.pig.backend.hadoop.executionengine.TaskContext;
import org.apache.pig.classification.InterfaceAudience;
import org.apache.pig.classification.InterfaceStability;
@@ -35,11 +33,7 @@ public class PigStatusReporter extends S
private TaskContext<?> context = null;
- static {
- JVMReuseManager.getInstance().registerForStaticDataCleanup(PigStatusReporter.class);
- }
-
- @StaticDataCleanup
+ //@StaticDataCleanup
public static void staticDataCleanup() {
reporter = null;
}