You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/05/29 20:59:17 UTC
svn commit: r1598370 - in /pig/branches/tez: ./ ivy/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
test/org/apache/pig/test/ test/org/apache/pig/tez/
Author: daijy
Date: Thu May 29 18:59:16 2014
New Revision: 1598370
URL: http://svn.apache.org/r1598370
Log:
PIG-3965: Fix MR unit tests on tez branch
Removed:
pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizer.java
Modified:
pig/branches/tez/ivy.xml
pig/branches/tez/ivy/libraries.properties
pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizerMR.java
pig/branches/tez/test/org/apache/pig/test/MiniGenericCluster.java
pig/branches/tez/test/org/apache/pig/test/TestCombiner.java
pig/branches/tez/test/org/apache/pig/test/TestSkewedJoin.java
pig/branches/tez/test/org/apache/pig/tez/TestTezLauncher.java
Modified: pig/branches/tez/ivy.xml
URL: http://svn.apache.org/viewvc/pig/branches/tez/ivy.xml?rev=1598370&r1=1598369&r2=1598370&view=diff
==============================================================================
--- pig/branches/tez/ivy.xml (original)
+++ pig/branches/tez/ivy.xml Thu May 29 18:59:16 2014
@@ -166,6 +166,8 @@
rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
<dependency org="org.apache.hadoop" name="hadoop-yarn-client"
rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
+ <dependency org="org.apache.hadoop" name="hadoop-yarn-server-applicationhistoryservice"
+ rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
<dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-hs"
rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
<dependency org="org.mortbay.jetty" name="jetty" rev="${jetty.version}"
Modified: pig/branches/tez/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/pig/branches/tez/ivy/libraries.properties?rev=1598370&r1=1598369&r2=1598370&view=diff
==============================================================================
--- pig/branches/tez/ivy/libraries.properties (original)
+++ pig/branches/tez/ivy/libraries.properties Thu May 29 18:59:16 2014
@@ -40,9 +40,9 @@ guava.version=11.0
jersey-core.version=1.8
hadoop-core.version=1.0.4
hadoop-test.version=1.0.4
-hadoop-common.version=2.3.0
-hadoop-hdfs.version=2.3.0
-hadoop-mapreduce.version=2.3.0
+hadoop-common.version=2.4.0
+hadoop-hdfs.version=2.4.0
+hadoop-mapreduce.version=2.4.0
hbase94.version=0.94.1
hbase95.version=0.96.0-hadoop1
hsqldb.version=1.8.0.10
Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizerMR.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizerMR.java?rev=1598370&r1=1598369&r2=1598370&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizerMR.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizerMR.java Thu May 29 18:59:16 2014
@@ -48,6 +48,10 @@ public class SecondaryKeyOptimizerMR ext
if (mr.isGlobalSort())
return;
+ // Don't optimize when we already have a custom partitioner
+ if (mr.getCustomPartitioner()!=null)
+ return;
+
info = SecondaryKeyOptimizerUtil.applySecondaryKeySort(mr.mapPlan, mr.reducePlan);
if (info != null && info.isUseSecondaryKey()) {
mr.setUseSecondaryKey(true);
Modified: pig/branches/tez/test/org/apache/pig/test/MiniGenericCluster.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/MiniGenericCluster.java?rev=1598370&r1=1598369&r2=1598370&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/MiniGenericCluster.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/MiniGenericCluster.java Thu May 29 18:59:16 2014
@@ -41,7 +41,6 @@ abstract public class MiniGenericCluster
protected MiniDFSCluster m_dfs = null;
protected FileSystem m_fileSys = null;
protected Configuration m_conf = null;
- protected Properties m_props = null;
protected static MiniGenericCluster INSTANCE = null;
protected static boolean isSetup = false;
@@ -102,7 +101,6 @@ abstract public class MiniGenericCluster
shutdownMiniDfsClusters();
shutdownMiniMrClusters();
m_conf = null;
- m_props = null;
}
protected void shutdownMiniDfsClusters() {
@@ -120,10 +118,7 @@ abstract public class MiniGenericCluster
public Properties getProperties() {
errorIfNotSetup();
- if (m_props == null) {
- m_props = ConfigurationUtil.toProperties(m_conf);
- }
- return m_props;
+ return ConfigurationUtil.toProperties(m_conf);
}
public Configuration getConfiguration() {
Modified: pig/branches/tez/test/org/apache/pig/test/TestCombiner.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestCombiner.java?rev=1598370&r1=1598369&r2=1598370&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestCombiner.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestCombiner.java Thu May 29 18:59:16 2014
@@ -173,6 +173,7 @@ public class TestCombiner {
properties.setProperty("io.sort.mb", "1");
PigServer pigServer = new PigServer(cluster.getExecType(), properties);
+ pigServer.getPigContext().getProperties().setProperty("mapred.child.java.opts", "-Xmx1024m");
pigServer.registerQuery("a = load 'MultiCombinerUseInput.txt' as (x:int);");
pigServer.registerQuery("b = group a all;");
pigServer.registerQuery("c = foreach b generate COUNT(a), SUM(a.$0), " +
Modified: pig/branches/tez/test/org/apache/pig/test/TestSkewedJoin.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestSkewedJoin.java?rev=1598370&r1=1598369&r2=1598370&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestSkewedJoin.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestSkewedJoin.java Thu May 29 18:59:16 2014
@@ -38,6 +38,11 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.lib.input.InvalidInputException;
+import org.apache.log4j.FileAppender;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import org.apache.log4j.SimpleLayout;
import org.apache.pig.PigServer;
import org.apache.pig.data.BagFactory;
import org.apache.pig.data.DataBag;
@@ -46,6 +51,7 @@ import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.builtin.PartitionSkewedKeys;
import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.newplan.logical.rules.ColumnPruneVisitor;
import org.apache.pig.test.utils.TestHelper;
import org.junit.AfterClass;
import org.junit.Before;
@@ -545,8 +551,12 @@ public class TestSkewedJoin {
"joined = JOIN exists BY a, missing BY a USING 'skewed';";
String logFile = Util.createTempFileDelOnExit("tmp", ".log").getAbsolutePath();
- String oldValue = (String) properties.setProperty("pig.logfile", logFile);
-
+ Logger logger = Logger.getLogger("org.apache.pig");
+ logger.setLevel(Level.INFO);
+ SimpleLayout layout = new SimpleLayout();
+ FileAppender appender = new FileAppender(layout, logFile.toString(), false, false, 0);
+ logger.addAppender(appender);
+
try {
pigServer.registerScript(new ByteArrayInputStream(script.getBytes("UTF-8")));
pigServer.openIterator("joined");
@@ -584,9 +594,7 @@ public class TestSkewedJoin {
assertTrue("This exception was not caused by InvalidInputException: " + e,
foundInvalidInputException);
} finally {
- if (oldValue != null) {
- properties.setProperty("pig.logfile", oldValue);
- }
+ LogManager.resetConfiguration();
}
}
Modified: pig/branches/tez/test/org/apache/pig/tez/TestTezLauncher.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/tez/TestTezLauncher.java?rev=1598370&r1=1598369&r2=1598370&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/tez/TestTezLauncher.java (original)
+++ pig/branches/tez/test/org/apache/pig/tez/TestTezLauncher.java Thu May 29 18:59:16 2014
@@ -60,7 +60,7 @@ public class TestTezLauncher {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
- cluster = MiniGenericCluster.buildCluster();
+ cluster = MiniGenericCluster.buildCluster(MiniGenericCluster.EXECTYPE_TEZ);
pc = new PigContext(new TezExecType(), cluster.getProperties());
Util.createInputFile(cluster, INPUT_FILE, INPUT_RECORDS);
}