You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/09/21 06:29:52 UTC
svn commit: r1626515 - in /pig/trunk: ./
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/
src/org/apache/pig/backend/hadoop/executionengine/util/ src/org/apache/pig...
Author: daijy
Date: Sun Sep 21 04:29:52 2014
New Revision: 1626515
URL: http://svn.apache.org/r1626515
Log:
PIG-4187: Fix Orc e2e tests
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PhyPlanSetter.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhyPlanVisitor.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/UdfCacheShipFilesVisitor.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java
pig/trunk/src/org/apache/pig/builtin/OrcStorage.java
pig/trunk/test/org/apache/pig/test/TestLoaderStorerShipCacheFiles.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1626515&r1=1626514&r2=1626515&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Sun Sep 21 04:29:52 2014
@@ -78,6 +78,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-4187: Fix Orc e2e tests (daijy)
+
PIG-4177: BigData_1 fail after PIG-4149 (daijy)
PIG-3507: Pig fails to run in local mode on a Kerberos enabled Hadoop cluster (kellyzly via rohini)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PhyPlanSetter.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PhyPlanSetter.java?rev=1626515&r1=1626514&r2=1626515&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PhyPlanSetter.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PhyPlanSetter.java Sun Sep 21 04:29:52 2014
@@ -349,6 +349,7 @@ public class PhyPlanSetter extends PhyPl
@Override
public void visitPreCombinerLocalRearrange(
POPreCombinerLocalRearrange preCombinerLocalRearrange) throws VisitorException {
+ super.visitPreCombinerLocalRearrange(preCombinerLocalRearrange);
preCombinerLocalRearrange.setParentPlan(parent);
}
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhyPlanVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhyPlanVisitor.java?rev=1626515&r1=1626514&r2=1626515&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhyPlanVisitor.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhyPlanVisitor.java Sun Sep 21 04:29:52 2014
@@ -347,7 +347,12 @@ public class PhyPlanVisitor extends Plan
*/
public void visitPreCombinerLocalRearrange(
POPreCombinerLocalRearrange preCombinerLocalRearrange) throws VisitorException {
- // TODO Auto-generated method stub
+ List<PhysicalPlan> inpPlans = preCombinerLocalRearrange.getPlans();
+ for (PhysicalPlan plan : inpPlans) {
+ pushWalker(mCurrentWalker.spawnChildWalker(plan));
+ visit();
+ popWalker();
+ }
}
public void visitPartialAgg(POPartialAgg poPartialAgg) throws VisitorException {
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/UdfCacheShipFilesVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/UdfCacheShipFilesVisitor.java?rev=1626515&r1=1626514&r2=1626515&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/UdfCacheShipFilesVisitor.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/UdfCacheShipFilesVisitor.java Sun Sep 21 04:29:52 2014
@@ -20,10 +20,13 @@ package org.apache.pig.backend.hadoop.ex
import java.util.HashSet;
import java.util.Set;
+import org.apache.pig.LoadFunc;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POCast;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
+import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.plan.DepthFirstWalker;
import org.apache.pig.impl.plan.VisitorException;
@@ -64,6 +67,22 @@ public class UdfCacheShipFilesVisitor ex
}
}
+ @Override
+ public void visitCast(POCast cast) {
+ if (cast.getFuncSpec()!=null) {
+ Object obj = PigContext.instantiateFuncFromSpec(cast.getFuncSpec());
+ if (obj instanceof LoadFunc) {
+ LoadFunc loadFunc = (LoadFunc)obj;
+ if (loadFunc.getCacheFiles()!=null) {
+ cacheFiles.addAll(loadFunc.getCacheFiles());
+ }
+ if (loadFunc.getShipFiles()!=null) {
+ shipFiles.addAll(loadFunc.getShipFiles());
+ }
+ }
+ }
+ }
+
public Set<String> getCacheFiles() {
return cacheFiles;
}
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java?rev=1626515&r1=1626514&r2=1626515&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/util/MapRedUtil.java Sun Sep 21 04:29:52 2014
@@ -172,7 +172,7 @@ public class MapRedUtil {
UDFContext udfc = UDFContext.getUDFContext();
udfc.addJobConf(job);
// don't deserialize in front-end
- if (udfc.isUDFConfEmpty()) {
+ if (!udfc.isFrontend()) {
udfc.deserialize();
}
}
Modified: pig/trunk/src/org/apache/pig/builtin/OrcStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/builtin/OrcStorage.java?rev=1626515&r1=1626514&r2=1626515&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/builtin/OrcStorage.java (original)
+++ pig/trunk/src/org/apache/pig/builtin/OrcStorage.java Sun Sep 21 04:29:52 2014
@@ -94,6 +94,7 @@ import org.apache.pig.impl.util.Utils;
import org.apache.pig.impl.util.orc.OrcUtils;
import org.joda.time.DateTime;
+import com.esotericsoftware.kryo.io.Input;
import com.google.common.annotations.VisibleForTesting;
/**
@@ -400,7 +401,8 @@ public class OrcStorage extends LoadFunc
throw new RuntimeException("Cannot find Hadoop" + hadoopVersion + "ShimsClass in classpath");
}
Class[] classList = new Class[] {OrcFile.class, HiveConf.class, AbstractSerDe.class,
- org.apache.hadoop.hive.shims.HadoopShims.class, HadoopShimsSecure.class, hadoopVersionShimsClass};
+ org.apache.hadoop.hive.shims.HadoopShims.class, HadoopShimsSecure.class, hadoopVersionShimsClass,
+ Input.class};
return FuncUtils.getShipFiles(classList);
}
Modified: pig/trunk/test/org/apache/pig/test/TestLoaderStorerShipCacheFiles.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestLoaderStorerShipCacheFiles.java?rev=1626515&r1=1626514&r2=1626515&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestLoaderStorerShipCacheFiles.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestLoaderStorerShipCacheFiles.java Sun Sep 21 04:29:52 2014
@@ -50,9 +50,10 @@ public abstract class TestLoaderStorerSh
hadoopVersion = "23";
}
String[] expectedJars = new String[] {"hive-common", "hive-exec", "hive-serde",
- "hive-shims-0." + hadoopVersion, "hive-shims-common-0", "hive-shims-common-secure"};
+ "hive-shims-0." + hadoopVersion, "hive-shims-common-0", "hive-shims-common-secure",
+ "kryo"};
- checkPlan(pp, expectedJars, 6, pigServer.getPigContext());
+ checkPlan(pp, expectedJars, 7, pigServer.getPigContext());
}
@Test
@@ -66,9 +67,10 @@ public abstract class TestLoaderStorerSh
hadoopVersion = "23";
}
String[] expectedJars = new String[] {"hive-common", "hive-exec", "hive-serde",
- "hive-shims-0." + hadoopVersion, "hive-shims-common-0", "hive-shims-common-secure"};
+ "hive-shims-0." + hadoopVersion, "hive-shims-common-0", "hive-shims-common-secure",
+ "kryo"};
- checkPlan(pp, expectedJars, 6, pigServer.getPigContext());
+ checkPlan(pp, expectedJars, 7, pigServer.getPigContext());
}
@Test