You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2017/04/07 05:12:12 UTC
svn commit: r1790504 - in /pig/trunk: ./ src/org/apache/pig/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/backend/hadoop/executionengine/tez/
test/org/apache/pig/test/
Author: daijy
Date: Fri Apr 7 05:12:12 2017
New Revision: 1790504
URL: http://svn.apache.org/viewvc?rev=1790504&view=rev
Log:
PIG-5210: Option to print MR/Tez plan before launching
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/PigConfiguration.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java
pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1790504&r1=1790503&r2=1790504&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Apr 7 05:12:12 2017
@@ -36,6 +36,8 @@ PIG-5067: Revisit union on numeric type
IMPROVEMENTS
+PIG-5210: Option to print MR/Tez plan before launching (ly16 via daijy)
+
PIG-5175: Upgrade jruby to 1.7.26 (daijy)
PIG-5110: Removing schema alias and :: coming from parent relation (szita via rohini)
Modified: pig/trunk/src/org/apache/pig/PigConfiguration.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigConfiguration.java?rev=1790504&r1=1790503&r2=1790504&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigConfiguration.java (original)
+++ pig/trunk/src/org/apache/pig/PigConfiguration.java Fri Apr 7 05:12:12 2017
@@ -508,6 +508,8 @@ public class PigConfiguration {
public static final String PIG_STORE_SCHEMA_DISAMBIGUATE_DEFAULT = "true";
+ public static final String PIG_PRINT_EXEC_PLAN = "pig.print.exec.plan";
+
// Deprecated settings of Pig 0.13
/**
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1790504&r1=1790503&r2=1790504&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java Fri Apr 7 05:12:12 2017
@@ -321,6 +321,12 @@ public class JobControlCompiler{
if(mro instanceof NativeMapReduceOper) {
return null;
}
+
+ //Print MR plan before launching if needed
+ if (conf.getBoolean(PigConfiguration.PIG_PRINT_EXEC_PLAN, false)) {
+ log.info(mro.toString());
+ }
+
Job job = getJob(plan, mro, conf, pigContext);
jobMroMap.put(job, mro);
jobCtrl.addJob(job);
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java?rev=1790504&r1=1790503&r2=1790504&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java Fri Apr 7 05:12:12 2017
@@ -30,6 +30,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.pig.PigConfiguration;
import org.apache.pig.PigException;
import org.apache.pig.backend.hadoop.PigATSClient;
import org.apache.pig.backend.hadoop.executionengine.JobCreationException;
@@ -110,6 +111,12 @@ public class TezJobCompiler {
for (Map.Entry<String, LocalResource> entry : localResources.entrySet()) {
log.info("Local resource: " + entry.getKey());
}
+
+ // Print Tez plan before launching if needed
+ if (conf.getBoolean(PigConfiguration.PIG_PRINT_EXEC_PLAN, false)) {
+ log.info(tezPlanNode.getTezOperPlan());
+ }
+
DAG tezDag = buildDAG(tezPlanNode, localResources);
tezDag.setDAGInfo(createDagInfo(TezScriptState.get().getScript()));
// set Tez caller context
Modified: pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java?rev=1790504&r1=1790503&r2=1790504&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java Fri Apr 7 05:12:12 2017
@@ -17,6 +17,8 @@
*/
package org.apache.pig.test;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@@ -25,6 +27,7 @@ import java.io.PrintStream;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -37,6 +40,12 @@ import java.util.StringTokenizer;
import junit.framework.Assert;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.log4j.Appender;
+import org.apache.log4j.FileAppender;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.SimpleLayout;
+import org.apache.log4j.WriterAppender;
import org.apache.pig.ComparisonFunc;
import org.apache.pig.EvalFunc;
import org.apache.pig.FuncSpec;
@@ -58,6 +67,7 @@ import org.apache.pig.impl.logicalLayer.
import org.apache.pig.impl.util.Pair;
import org.apache.pig.impl.util.UDFContext;
import org.apache.pig.impl.util.Utils;
+import org.apache.pig.newplan.logical.rules.ColumnPruneVisitor;
import org.apache.pig.test.utils.Identity;
import org.junit.Assume;
import org.junit.Before;
@@ -1251,7 +1261,17 @@ public class TestEvalPipelineLocal {
@Test
public void testBytesRawComparatorDesc() throws Exception{
File f1 = createFile(new String[]{"2", "1", "4", "3"});
-
+
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ Logger logger = Logger.getRootLogger();
+
+ logger.setLevel(Level.INFO);
+ SimpleLayout layout = new SimpleLayout();
+ Appender appender = new WriterAppender(layout, new PrintStream(bos));
+ logger.addAppender(appender);
+
+ // Also test PIG-5210 here in the same test
+ pigServer.getPigContext().getProperties().setProperty("pig.print.exec.plan", "true");
pigServer.registerQuery("a = load '" + Util.generateURI(f1.toString(), pigServer.getPigContext())
+ "' as (value:long);");
pigServer.registerQuery("b = foreach a generate " + TOTUPLENOINNERSCHEMA.class.getName() + "(value);");
@@ -1264,5 +1284,9 @@ public class TestEvalPipelineLocal {
Assert.assertEquals(iter.next().toString(), "(2)");
Assert.assertEquals(iter.next().toString(), "(1)");
Assert.assertFalse(iter.hasNext());
+
+ logger.removeAppender(appender);
+
+ Assert.assertTrue(bos.toString().contains("New For Each(false,false)[tuple]"));
}
}