You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/05/29 20:59:17 UTC

svn commit: r1598370 - in /pig/branches/tez: ./ ivy/ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ test/org/apache/pig/test/ test/org/apache/pig/tez/

Author: daijy
Date: Thu May 29 18:59:16 2014
New Revision: 1598370

URL: http://svn.apache.org/r1598370
Log:
PIG-3965: Fix MR unit tests on tez branch

Removed:
    pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizer.java
Modified:
    pig/branches/tez/ivy.xml
    pig/branches/tez/ivy/libraries.properties
    pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizerMR.java
    pig/branches/tez/test/org/apache/pig/test/MiniGenericCluster.java
    pig/branches/tez/test/org/apache/pig/test/TestCombiner.java
    pig/branches/tez/test/org/apache/pig/test/TestSkewedJoin.java
    pig/branches/tez/test/org/apache/pig/tez/TestTezLauncher.java

Modified: pig/branches/tez/ivy.xml
URL: http://svn.apache.org/viewvc/pig/branches/tez/ivy.xml?rev=1598370&r1=1598369&r2=1598370&view=diff
==============================================================================
--- pig/branches/tez/ivy.xml (original)
+++ pig/branches/tez/ivy.xml Thu May 29 18:59:16 2014
@@ -166,6 +166,8 @@
       rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
     <dependency org="org.apache.hadoop" name="hadoop-yarn-client" 
       rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
+    <dependency org="org.apache.hadoop" name="hadoop-yarn-server-applicationhistoryservice" 
+      rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
     <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-hs" 
       rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
     <dependency org="org.mortbay.jetty" name="jetty" rev="${jetty.version}"

Modified: pig/branches/tez/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/pig/branches/tez/ivy/libraries.properties?rev=1598370&r1=1598369&r2=1598370&view=diff
==============================================================================
--- pig/branches/tez/ivy/libraries.properties (original)
+++ pig/branches/tez/ivy/libraries.properties Thu May 29 18:59:16 2014
@@ -40,9 +40,9 @@ guava.version=11.0
 jersey-core.version=1.8
 hadoop-core.version=1.0.4
 hadoop-test.version=1.0.4
-hadoop-common.version=2.3.0
-hadoop-hdfs.version=2.3.0
-hadoop-mapreduce.version=2.3.0
+hadoop-common.version=2.4.0
+hadoop-hdfs.version=2.4.0
+hadoop-mapreduce.version=2.4.0
 hbase94.version=0.94.1
 hbase95.version=0.96.0-hadoop1
 hsqldb.version=1.8.0.10

Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizerMR.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizerMR.java?rev=1598370&r1=1598369&r2=1598370&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizerMR.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizerMR.java Thu May 29 18:59:16 2014
@@ -48,6 +48,10 @@ public class SecondaryKeyOptimizerMR ext
         if (mr.isGlobalSort())
             return;
 
+        // Don't optimize when we already have a custom partitioner
+        if (mr.getCustomPartitioner()!=null)
+            return;
+
         info = SecondaryKeyOptimizerUtil.applySecondaryKeySort(mr.mapPlan, mr.reducePlan);
         if (info != null && info.isUseSecondaryKey()) {
             mr.setUseSecondaryKey(true);

Modified: pig/branches/tez/test/org/apache/pig/test/MiniGenericCluster.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/MiniGenericCluster.java?rev=1598370&r1=1598369&r2=1598370&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/MiniGenericCluster.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/MiniGenericCluster.java Thu May 29 18:59:16 2014
@@ -41,7 +41,6 @@ abstract public class MiniGenericCluster
     protected MiniDFSCluster m_dfs = null;
     protected FileSystem m_fileSys = null;
     protected Configuration m_conf = null;
-    protected Properties m_props = null;
 
     protected static MiniGenericCluster INSTANCE = null;
     protected static boolean isSetup = false;
@@ -102,7 +101,6 @@ abstract public class MiniGenericCluster
         shutdownMiniDfsClusters();
         shutdownMiniMrClusters();
         m_conf = null;
-        m_props = null;
     }
 
     protected void shutdownMiniDfsClusters() {
@@ -120,10 +118,7 @@ abstract public class MiniGenericCluster
 
     public Properties getProperties() {
         errorIfNotSetup();
-        if (m_props == null) {
-            m_props = ConfigurationUtil.toProperties(m_conf);
-        }
-        return m_props;
+        return ConfigurationUtil.toProperties(m_conf);
     }
 
     public Configuration getConfiguration() {

Modified: pig/branches/tez/test/org/apache/pig/test/TestCombiner.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestCombiner.java?rev=1598370&r1=1598369&r2=1598370&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestCombiner.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestCombiner.java Thu May 29 18:59:16 2014
@@ -173,6 +173,7 @@ public class TestCombiner {
         properties.setProperty("io.sort.mb", "1");
 
         PigServer pigServer = new PigServer(cluster.getExecType(), properties);
+        pigServer.getPigContext().getProperties().setProperty("mapred.child.java.opts", "-Xmx1024m");
         pigServer.registerQuery("a = load 'MultiCombinerUseInput.txt' as (x:int);");
         pigServer.registerQuery("b = group a all;");
         pigServer.registerQuery("c = foreach b generate COUNT(a), SUM(a.$0), " +

Modified: pig/branches/tez/test/org/apache/pig/test/TestSkewedJoin.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestSkewedJoin.java?rev=1598370&r1=1598369&r2=1598370&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestSkewedJoin.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestSkewedJoin.java Thu May 29 18:59:16 2014
@@ -38,6 +38,11 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapreduce.lib.input.InvalidInputException;
+import org.apache.log4j.FileAppender;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import org.apache.log4j.SimpleLayout;
 import org.apache.pig.PigServer;
 import org.apache.pig.data.BagFactory;
 import org.apache.pig.data.DataBag;
@@ -46,6 +51,7 @@ import org.apache.pig.data.Tuple;
 import org.apache.pig.data.TupleFactory;
 import org.apache.pig.impl.builtin.PartitionSkewedKeys;
 import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.newplan.logical.rules.ColumnPruneVisitor;
 import org.apache.pig.test.utils.TestHelper;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -545,8 +551,12 @@ public class TestSkewedJoin {
           "joined = JOIN exists BY a, missing BY a USING 'skewed';";
 
         String logFile = Util.createTempFileDelOnExit("tmp", ".log").getAbsolutePath();
-        String oldValue = (String) properties.setProperty("pig.logfile", logFile);
-
+        Logger logger = Logger.getLogger("org.apache.pig");
+        logger.setLevel(Level.INFO);
+        SimpleLayout layout = new SimpleLayout();
+        FileAppender appender = new FileAppender(layout, logFile.toString(), false, false, 0);
+        logger.addAppender(appender);
+        
         try {
             pigServer.registerScript(new ByteArrayInputStream(script.getBytes("UTF-8")));
             pigServer.openIterator("joined");
@@ -584,9 +594,7 @@ public class TestSkewedJoin {
             assertTrue("This exception was not caused by InvalidInputException: " + e,
                     foundInvalidInputException);
         } finally {
-            if (oldValue != null) {
-                properties.setProperty("pig.logfile", oldValue);
-            }
+            LogManager.resetConfiguration();
         }
     }
 

Modified: pig/branches/tez/test/org/apache/pig/tez/TestTezLauncher.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/tez/TestTezLauncher.java?rev=1598370&r1=1598369&r2=1598370&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/tez/TestTezLauncher.java (original)
+++ pig/branches/tez/test/org/apache/pig/tez/TestTezLauncher.java Thu May 29 18:59:16 2014
@@ -60,7 +60,7 @@ public class TestTezLauncher {
 
     @BeforeClass
     public static void setUpBeforeClass() throws Exception {
-        cluster = MiniGenericCluster.buildCluster();
+        cluster = MiniGenericCluster.buildCluster(MiniGenericCluster.EXECTYPE_TEZ);
         pc = new PigContext(new TezExecType(), cluster.getProperties());
         Util.createInputFile(cluster, INPUT_FILE, INPUT_RECORDS);
     }