You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by xu...@apache.org on 2016/03/04 19:17:47 UTC
svn commit: r1733627 [17/18] - in /pig/branches/spark: ./ bin/ conf/
contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/
contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/
contrib/piggybank/java/src/mai...
Modified: pig/branches/spark/test/org/apache/pig/test/TestSkewedJoin.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestSkewedJoin.java?rev=1733627&r1=1733626&r2=1733627&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestSkewedJoin.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestSkewedJoin.java Fri Mar 4 18:17:39 2016
@@ -51,7 +51,6 @@ import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.builtin.PartitionSkewedKeys;
import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.newplan.logical.rules.ColumnPruneVisitor;
import org.apache.pig.test.utils.TestHelper;
import org.junit.AfterClass;
import org.junit.Before;
@@ -66,8 +65,9 @@ public class TestSkewedJoin {
private static final String INPUT_FILE5 = "SkewedJoinInput5.txt";
private static final String INPUT_FILE6 = "SkewedJoinInput6.txt";
private static final String INPUT_FILE7 = "SkewedJoinInput7.txt";
- private static final String INPUT_DIR = "build/test/data";
- private static final String OUTPUT_DIR = "build/test/output";
+ private static final String TEST_DIR = Util.getTestDirectory(TestSkewedJoin.class);
+ private static final String INPUT_DIR = TEST_DIR + Path.SEPARATOR + "input";
+ private static final String OUTPUT_DIR = TEST_DIR + Path.SEPARATOR + "output";
private static FileSystem fs;
private static PigServer pigServer;
@@ -97,8 +97,8 @@ public class TestSkewedJoin {
}
private static void createFiles() throws IOException {
- new File(INPUT_DIR).mkdir();
- new File(OUTPUT_DIR).mkdir();
+ new File(INPUT_DIR).mkdirs();
+ new File(OUTPUT_DIR).mkdirs();
PrintWriter w = new PrintWriter(new FileWriter(INPUT_DIR + "/" + INPUT_FILE1));
@@ -183,8 +183,7 @@ public class TestSkewedJoin {
}
private static void deleteFiles() throws IOException {
- Util.deleteDirectory(new File(INPUT_DIR));
- Util.deleteDirectory(new File(OUTPUT_DIR));
+ Util.deleteDirectory(new File(TEST_DIR));
}
@Test
@@ -563,7 +562,7 @@ public class TestSkewedJoin {
SimpleLayout layout = new SimpleLayout();
FileAppender appender = new FileAppender(layout, logFile.toString(), false, false, 0);
logger.addAppender(appender);
-
+
try {
pigServer.registerScript(new ByteArrayInputStream(script.getBytes("UTF-8")));
pigServer.openIterator("joined");
Modified: pig/branches/spark/test/org/apache/pig/test/TestStreaming.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestStreaming.java?rev=1733627&r1=1733626&r2=1733627&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestStreaming.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestStreaming.java Fri Mar 4 18:17:39 2016
@@ -40,14 +40,23 @@ import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
public class TestStreaming {
private static final MiniGenericCluster cluster = MiniGenericCluster.buildCluster();
+ private static File testDataDir;
private PigServer pigServer;
+ @BeforeClass
+ public static void oneTimeSetUp() throws Exception {
+ // Create the test data directory if needed
+ testDataDir = new File(Util.getTestDirectory(TestStreaming.class));
+ testDataDir.mkdirs();
+ }
+
@Before
public void setup() throws ExecException {
pigServer = new PigServer(cluster.getExecType(), cluster.getProperties());
@@ -61,6 +70,7 @@ public class TestStreaming {
@AfterClass
public static void oneTimeTearDown() throws Exception {
cluster.shutDown();
+ Util.deleteDirectory(testDataDir);
}
private TupleFactory tf = TupleFactory.getInstance();
@@ -293,7 +303,9 @@ public class TestStreaming {
"}",
};
File command1 = Util.createInputFile("script", "pl", script);
- File command2 = Util.createInputFile("script", "pl", script);
+ // Test relative path
+ File command2 = new File(testDataDir, "testInputShipSpecs.pl");
+ Util.writeToFile(command2, script);
// Expected results
String[] expectedFirstFields =
Modified: pig/branches/spark/test/org/apache/pig/test/TestStringUDFs.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestStringUDFs.java?rev=1733627&r1=1733626&r2=1733627&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestStringUDFs.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestStringUDFs.java Fri Mar 4 18:17:39 2016
@@ -128,7 +128,10 @@ public class TestStringUDFs {
REPLACE replace = new REPLACE();
Tuple testTuple = Util.buildTuple("foobar", "z", "x");
assertEquals("foobar".replace("z", "x"), replace.exec(testTuple));
-
+
+ // Use cached version of pattern in REPLACE
+ assertEquals("foobar".replace("z", "x"), replace.exec(testTuple));
+
testTuple = Util.buildTuple("foobar", "oo", "aa");
assertEquals("foobar".replace("oo", "aa"), replace.exec(testTuple));
}
Modified: pig/branches/spark/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java?rev=1733627&r1=1733626&r2=1733627&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java Fri Mar 4 18:17:39 2016
@@ -745,8 +745,8 @@ public class TestTypeCheckingValidatorNe
// value of datetime and one of other type
LogicalExpressionPlan plan = new LogicalExpressionPlan();
ConstantExpression constant0 = new ConstantExpression(plan, new DateTime(0L));
- ConstantExpression constant1 = new ConstantExpression(plan, new DataByteArray("1970-01-01T00:00:00.000Z"));
- CastExpression cast1 = new CastExpression(plan, constant1, createFS(DataType.BYTEARRAY));
+ ConstantExpression constant1 = new ConstantExpression(plan, new String("1970-01-01T00:00:00.000Z"));
+ CastExpression cast1 = new CastExpression(plan, constant1, createFS(DataType.CHARARRAY));
EqualExpression eq1 = new EqualExpression(plan, constant0, cast1);
CompilationMessageCollector collector = new CompilationMessageCollector();
@@ -3685,6 +3685,7 @@ public class TestTypeCheckingValidatorNe
super(p, walker);
}
+ @Override
public void visit(CastExpression cExp){
casts.add(cExp);
}
Modified: pig/branches/spark/test/org/apache/pig/test/TestUDFContext.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestUDFContext.java?rev=1733627&r1=1733626&r2=1733627&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestUDFContext.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestUDFContext.java Fri Mar 4 18:17:39 2016
@@ -17,28 +17,44 @@
*/
package org.apache.pig.test;
+import static org.apache.pig.builtin.mock.Storage.resetData;
+import static org.apache.pig.builtin.mock.Storage.tuple;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.FileWriter;
+import java.io.IOException;
import java.util.Iterator;
+import java.util.List;
import java.util.Properties;
+import org.apache.hadoop.mapreduce.Job;
import org.apache.pig.PigServer;
+import org.apache.pig.ResourceSchema;
import org.apache.pig.builtin.PigStorage;
+import org.apache.pig.builtin.mock.Storage;
+import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.io.FileLocalizer;
+import org.apache.pig.impl.util.ObjectSerializer;
import org.apache.pig.impl.util.UDFContext;
+import org.junit.AfterClass;
import org.junit.Test;
+import com.google.common.io.Files;
+
public class TestUDFContext {
+ @AfterClass
+ public static void oneTimeTearDown() throws Exception {
+ FileLocalizer.deleteTempFiles();
+ }
+
@Test
public void testUDFContext() throws Exception {
- File a = Util.createLocalInputFile("a.txt", new String[] { "dumb" });
- File b = Util.createLocalInputFile("b.txt", new String[] { "dumber" });
- FileLocalizer.deleteTempFiles();
+ File a = Util.createInputFile("inp1", "txt", new String[] { "dumb" });
+ File b = Util.createInputFile("inp2", "txt", new String[] { "dumber" });
PigServer pig = new PigServer(Util.getLocalTestMode(), new Properties());
String[] statement = { "A = LOAD '" + Util.encodeEscape(a.getAbsolutePath()) +
"' USING org.apache.pig.test.utils.UDFContextTestLoader('joe');",
@@ -100,5 +116,112 @@ public class TestUDFContext {
}
+
+ @Test
+ public void testUDFContextSeparator() throws Exception {
+
+ File inputFile = Util.createInputFile("input", "txt", new String[] { "f1\tf2\tf3\tf4\tf5" });
+
+ PigServer pigServer = new PigServer(Util.getLocalTestMode(), new Properties());
+ Storage.Data data = resetData(pigServer);
+
+ String inputPath = Util.encodeEscape(inputFile.getAbsolutePath());
+ File file = Files.createTempDir();
+ file.deleteOnExit();
+ String outputFile = Util.generateURI(file.getPath() + File.pathSeparator + "out", pigServer.getPigContext());
+ String query = "A = LOAD '" + inputPath + "' USING PigStorage() as (f1, f2, f3, f4, f5);"
+ + "B = LOAD '" + inputPath + "' USING PigStorage();"
+ + "B = FOREACH B GENERATE $0, $1;"
+ + "C = LOAD '" + inputPath + "' USING " + FieldsByIndexLoader.class.getName() + "('1,2');"
+ // Scalar to force PigStorage to be always visited first in LoaderProcessor
+ + "C = FOREACH C GENERATE *, B.$0;"
+ + "STORE A INTO 'A' USING mock.Storage();"
+ + "STORE B INTO 'B' USING mock.Storage();"
+ + "STORE C INTO 'C' USING mock.Storage();"
+ + "STORE A INTO '" + outputFile + "' USING " + FieldsByIndexLoader.class.getName() + "();";
+
+ pigServer.registerQuery(query);
+
+ List<Tuple> a = data.get("A");
+ List<Tuple> b = data.get("B");
+ List<Tuple> c = data.get("C");
+ assertEquals(1, a.size());
+ assertEquals(1, b.size());
+ assertEquals(1, c.size());
+ DataByteArray f1 = new DataByteArray("f1");
+ DataByteArray f2 = new DataByteArray("f2");
+ DataByteArray f3 = new DataByteArray("f3");
+ DataByteArray f4 = new DataByteArray("f4");
+ DataByteArray f5 = new DataByteArray("f5");
+ assertEquals(tuple(f1, f2, f3, f4, f5), a.get(0));
+ assertEquals(tuple(f1, f2), b.get(0));
+ assertEquals(tuple(f2, f3, f1), c.get(0));
+ }
+
+
+ public static class FieldsByIndexLoader extends PigStorage {
+
+ // Tests the case of one user LoadFunc where UDF properties was a class variable
+ // and getSchema() was used to determined frontend instead of
+ // UDFContext.getUDFContext().isFrontEnd();
+ private boolean frontend = false;
+ private Properties props = UDFContext.getUDFContext().getUDFProperties(this.getClass());
+ private boolean[] selectedFields = new boolean[5]; //Assuming data always has 5 columns
+ private String storeSignature;
+
+ public FieldsByIndexLoader() {
+ }
+
+ public FieldsByIndexLoader(String fieldIndices) {
+ String[] requiredFields = fieldIndices.split(",");
+ for (String index : requiredFields) {
+ selectedFields[Integer.parseInt(index)] = true;
+ }
+ }
+
+ @Override
+ public void setLocation(String location, Job job) throws IOException {
+ if (frontend) {
+ // PigStorage should deserialize this as
+ // mRequiredColumns = (boolean[])ObjectSerializer.deserialize(p.getProperty(signature));
+ props.setProperty(signature, ObjectSerializer.serialize(selectedFields));
+ }
+ super.setLocation(location, job);
+ }
+
+ @Override
+ public ResourceSchema getSchema(String location, Job job)
+ throws IOException {
+ frontend = true;
+ return super.getSchema(location, job);
+ }
+
+ @Override
+ public void checkSchema(ResourceSchema s) throws IOException {
+ super.checkSchema(s);
+ UDFContext udfContext = UDFContext.getUDFContext();
+ Properties props = udfContext.getUDFProperties(this.getClass(), new String[]{storeSignature});
+ props.setProperty("testkey", "testvalue");
+ }
+
+ @Override
+ public void setStoreFuncUDFContextSignature(String signature) {
+ this.storeSignature = signature;
+ }
+
+ @Override
+ public void setStoreLocation(String location, Job job)
+ throws IOException {
+ if (!UDFContext.getUDFContext().isFrontend()) {
+ Properties udfProps = UDFContext.getUDFContext().getUDFProperties(this.getClass(), new String[]{storeSignature});
+ if (!("testvalue").equals(udfProps.getProperty("testkey"))) {
+ throw new IOException("UDFContext does not have expected values");
+ }
+ }
+ super.setStoreLocation(location, job);
+ }
+
+
+ }
}
Modified: pig/branches/spark/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/Util.java?rev=1733627&r1=1733626&r2=1733627&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/Util.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/Util.java Fri Mar 4 18:17:39 2016
@@ -118,6 +118,8 @@ public class Util {
public static final boolean WINDOWS /* borrowed from Path.WINDOWS, Shell.WINDOWS */
= System.getProperty("os.name").startsWith("Windows");
+ public static final String TEST_DIR = System.getProperty("test.build.dir", "build/test");
+
// Helper Functions
// =================
@@ -176,6 +178,28 @@ public class Util {
return t;
}
+ /**
+ * Create an array of tuple bags with specified size created by splitting
+ * the input array of primitive types
+ *
+ * @param input Array of primitive types
+ * @param bagSize The number of tuples to be split and copied into each bag
+ *
+ * @return an array of tuple bags with each bag containing bagSize tuples split from the input
+ */
+ static public <T> Tuple[] splitCreateBagOfTuples(T[] input, int bagSize)
+ throws ExecException {
+ List<Tuple> result = new ArrayList<Tuple>();
+ for (int from = 0; from < input.length; from += bagSize) {
+ Tuple t = TupleFactory.getInstance().newTuple(1);
+ int to = from + bagSize < input.length ? from + bagSize
+ : input.length;
+ T[] array = Arrays.copyOfRange(input, from, to);
+ result.add(loadNestTuple(t, array));
+ }
+ return result.toArray(new Tuple[0]);
+ }
+
static public <T>void addToTuple(Tuple t, T[] b)
{
for(int i = 0; i < b.length; i++)
@@ -934,6 +958,7 @@ public class Util {
thread = new Thread (this);
thread.start ();
}
+ @Override
public void run () {
try {
InputStreamReader isr = new InputStreamReader (is);
@@ -1392,37 +1417,47 @@ public class Util {
return ExecTypeProvider.fromString("local");
}
- public static void createLogAppender(Class clazz, String appenderName, Writer writer) {
- Logger logger = Logger.getLogger(clazz);
+ public static void createLogAppender(String appenderName, Writer writer, Class...clazzes) {
WriterAppender writerAppender = new WriterAppender(new PatternLayout("%d [%t] %-5p %c %x - %m%n"), writer);
writerAppender.setName(appenderName);
- logger.addAppender(writerAppender);
+ for (Class clazz : clazzes) {
+ Logger logger = Logger.getLogger(clazz);
+ logger.addAppender(writerAppender);
+ }
}
- public static void removeLogAppender(Class clazz, String appenderName) {
- Logger logger = Logger.getLogger(clazz);
- Appender appender = logger.getAppender(appenderName);
- appender.close();
- logger.removeAppender(appenderName);
+ public static void removeLogAppender(String appenderName, Class...clazzes) {
+ for (Class clazz : clazzes) {
+ Logger logger = Logger.getLogger(clazz);
+ Appender appender = logger.getAppender(appenderName);
+ appender.close();
+ logger.removeAppender(appenderName);
+ }
}
public static Path getFirstPartFile(Path path) throws Exception {
- FileStatus[] parts = FileSystem.get(path.toUri(), new Configuration()).listStatus(path,
+ FileStatus[] parts = FileSystem.get(path.toUri(), new Configuration()).listStatus(path,
new PathFilter() {
@Override
public boolean accept(Path path) {
return path.getName().startsWith("part-");
}
});
- return parts[0].getPath();
+ return parts[0].getPath();
}
public static File getFirstPartFile(File dir) throws Exception {
File[] parts = dir.listFiles(new FilenameFilter() {
+ @Override
public boolean accept(File dir, String name) {
return name.startsWith("part-");
};
});
- return parts[0];
+ return parts[0];
+ }
+
+ @SuppressWarnings("rawtypes")
+ public static String getTestDirectory(Class testClass) {
+ return TEST_DIR + Path.SEPARATOR + "testdata" + Path.SEPARATOR +testClass.getSimpleName();
}
}
Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld?rev=1733627&r1=1733626&r2=1733627&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld (original)
+++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Limit-2.gld Fri Mar 4 18:17:39 2016
@@ -4,86 +4,92 @@
#--------------------------------------------------
# TEZ DAG plan: pig-0_scope-0
#--------------------------------------------------
-Tez vertex scope-12 -> Tez vertex scope-22,Tez vertex scope-33,
-Tez vertex scope-22 -> Tez vertex scope-33,
-Tez vertex scope-33 -> Tez vertex scope-35,
-Tez vertex scope-35 -> Tez vertex scope-46,
-Tez vertex scope-46
+Tez vertex scope-12 -> Tez vertex scope-28,
+Tez vertex scope-28 -> Tez vertex scope-39,
+Tez vertex scope-39 -> Tez vertex scope-40,
+Tez vertex scope-40 -> Tez vertex scope-51,
+Tez vertex scope-51
Tez vertex scope-12
# Plan on vertex
-Local Rearrange[tuple]{tuple}(false) - scope-15 -> scope-22
+Local Rearrange[tuple]{tuple}(false) - scope-21 -> scope-28
| |
-| Constant(DummyVal) - scope-14
+| Constant(DummyVal) - scope-20
|
-|---New For Each(false,false,true)[tuple] - scope-21
+|---New For Each(false,false,true)[tuple] - scope-27
| |
| Project[int][0] - scope-8
| |
| Project[int][1] - scope-9
| |
- | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-20
+ | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-26
| |
- | |---Project[tuple][*] - scope-19
+ | |---Project[tuple][*] - scope-25
|
- |---ReservoirSample - scope-18
+ |---ReservoirSample - scope-24
|
- |---b: Local Rearrange[tuple]{tuple}(false) - scope-13 -> scope-33
+ |---a: New For Each(false,false)[bag] - scope-7
| |
- | Project[int][0] - scope-8
+ | Cast[int] - scope-2
| |
- | Project[int][1] - scope-9
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[int] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
|
- |---a: New For Each(false,false)[bag] - scope-7
- | |
- | Cast[int] - scope-2
- | |
- | |---Project[bytearray][0] - scope-1
- | |
- | Cast[int] - scope-5
- | |
- | |---Project[bytearray][1] - scope-4
- |
- |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
-Tez vertex scope-22
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-28
# Plan on vertex
-POValueOutputTez - scope-32 -> [scope-33]
+POValueOutputTez - scope-38 -> [scope-39]
|
-|---New For Each(false)[tuple] - scope-31
+|---New For Each(false)[tuple] - scope-37
| |
- | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.FindQuantilesTez)[tuple] - scope-30
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.FindQuantilesTez)[tuple] - scope-36
| |
- | |---Project[tuple][*] - scope-29
+ | |---Project[tuple][*] - scope-35
|
- |---New For Each(false,false)[tuple] - scope-28
+ |---New For Each(false,false)[tuple] - scope-34
| |
- | Constant(-1) - scope-27
+ | Constant(-1) - scope-33
| |
- | Project[bag][1] - scope-24
+ | Project[bag][1] - scope-30
|
- |---Package(Packager)[tuple]{bytearray} - scope-23
-Tez vertex scope-33
+ |---Package(Packager)[tuple]{bytearray} - scope-29
+Tez vertex scope-39
# Plan on vertex
-POIdentityInOutTez - scope-34 <- scope-12 -> scope-35
+b: Local Rearrange[tuple]{tuple}(false) - scope-13 -> scope-40
| |
| Project[int][0] - scope-8
| |
| Project[int][1] - scope-9
-Tez vertex scope-35
+|
+|---a: New For Each(false,false)[bag] - scope-19
+ | |
+ | Cast[int] - scope-16
+ | |
+ | |---Project[bytearray][0] - scope-15
+ | |
+ | Cast[int] - scope-18
+ | |
+ | |---Project[bytearray][1] - scope-17
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-14
+Tez vertex scope-40
# Plan on vertex
-POValueOutputTez - scope-45 -> [scope-46]
+POValueOutputTez - scope-50 -> [scope-51]
|
-|---Limit - scope-44
+|---Limit - scope-49
|
- |---New For Each(true)[tuple] - scope-43
+ |---New For Each(true)[tuple] - scope-48
| |
- | Project[bag][1] - scope-42
+ | Project[bag][1] - scope-47
|
- |---Package(LitePackager)[tuple]{tuple} - scope-41
-Tez vertex scope-46
+ |---Package(LitePackager)[tuple]{tuple} - scope-46
+Tez vertex scope-51
# Plan on vertex
c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-11
|
-|---Limit - scope-48
+|---Limit - scope-53
|
- |---POValueInputTez - scope-47 <- scope-35
+ |---POValueInputTez - scope-52 <- scope-40
Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld?rev=1733627&r1=1733626&r2=1733627&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld (original)
+++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld Fri Mar 4 18:17:39 2016
@@ -294,12 +294,12 @@ f1: Store(file:///tmp/output/f1:org.apac
|---POValueInputTez - scope-383 <- scope-378
Tez vertex scope-384
# Plan on vertex
-f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-413
+f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-413 -> scope-316
|
|---POValueInputTez - scope-385 <- scope-341
Tez vertex scope-386
# Plan on vertex
-f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-414
+f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-414 -> scope-316
|
|---POValueInputTez - scope-387 <- scope-378
Tez vertex group scope-412 <- [scope-384, scope-386] -> null
Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld?rev=1733627&r1=1733626&r2=1733627&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld (original)
+++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld Fri Mar 4 18:17:39 2016
@@ -4,9 +4,8 @@
#--------------------------------------------------
# TEZ DAG plan: pig-0_scope-0
#--------------------------------------------------
-Tez vertex scope-106 -> Tez vertex scope-113,Tez vertex scope-119,Tez vertex scope-126,Tez vertex scope-148,Tez vertex scope-158,Tez vertex scope-167,Tez vertex group scope-208,
+Tez vertex scope-106 -> Tez vertex scope-113,Tez vertex scope-126,Tez vertex scope-129,Tez vertex scope-148,Tez vertex scope-158,Tez vertex scope-167,Tez vertex group scope-208,
Tez vertex scope-113
-Tez vertex scope-119 -> Tez vertex scope-126,Tez vertex scope-129,
Tez vertex scope-126
Tez vertex scope-129
Tez vertex scope-148 -> Tez vertex scope-158,
@@ -77,11 +76,41 @@ Tez vertex scope-106
| | |
| | |---Constant(5) - scope-11
| |
+| c: Split - scope-205
+| | |
+| | c1: Local Rearrange[tuple]{int}(false) - scope-47 -> scope-126
+| | | |
+| | | Project[int][0] - scope-48
+| | |
+| | c2: Local Rearrange[tuple]{int}(false) - scope-193 -> scope-129
+| | | |
+| | | Project[int][0] - scope-195
+| | |
+| | |---c3: New For Each(false,false)[bag] - scope-181
+| | | |
+| | | Project[int][0] - scope-182
+| | | |
+| | | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-183
+| | | |
+| | | |---Project[bag][0] - scope-184
+| | | |
+| | | |---Project[bag][1] - scope-185
+| | |
+| | |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-196
+| |
+| |---c: Filter[bag] - scope-34
+| | |
+| | Less Than or Equal[boolean] - scope-37
+| | |
+| | |---Project[int][0] - scope-35
+| | |
+| | |---Constant(10) - scope-36
+| |
| d1: Split - scope-206
| | |
| | d1: Store(file:///tmp/output/d1:org.apache.pig.builtin.PigStorage) - scope-80
| | |
-| | f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-209
+| | f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-209 -> scope-105
| |
| |---d1: Filter[bag] - scope-73
| | |
@@ -98,8 +127,6 @@ Tez vertex scope-106
| | |---Project[int][0] - scope-70
| | |
| | |---Constant(10) - scope-71
-| |
-| POValueOutputTez - scope-107 -> [scope-119]
|
|---a: New For Each(false,false)[bag] - scope-7
| |
@@ -131,39 +158,6 @@ b1: Split - scope-201
| | |---Project[bag][1] - scope-28
|
|---b1: Package(Packager)[tuple]{int} - scope-17
-Tez vertex scope-119
-# Plan on vertex
-c: Split - scope-205
-| |
-| c1: Local Rearrange[tuple]{int}(false) - scope-47 -> scope-126
-| | |
-| | Project[int][0] - scope-48
-| |
-| c2: Local Rearrange[tuple]{int}(false) - scope-193 -> scope-129
-| | |
-| | Project[int][0] - scope-195
-| |
-| |---c3: New For Each(false,false)[bag] - scope-181
-| | |
-| | Project[int][0] - scope-182
-| | |
-| | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-183
-| | |
-| | |---Project[bag][0] - scope-184
-| | |
-| | |---Project[bag][1] - scope-185
-| |
-| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-196
-|
-|---c: Filter[bag] - scope-34
- | |
- | Less Than or Equal[boolean] - scope-37
- | |
- | |---Project[int][0] - scope-35
- | |
- | |---Constant(10) - scope-36
- |
- |---POValueInputTez - scope-120 <- scope-106
Tez vertex scope-126
# Plan on vertex
c1: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-54
@@ -176,7 +170,7 @@ c1: Store(file:///tmp/output/c1:org.apac
|
|---c1: Package(Packager)[tuple]{int} - scope-46
Tez vertex scope-129
-# Combine plan on edge <scope-119>
+# Combine plan on edge <scope-106>
c2: Local Rearrange[tuple]{int}(false) - scope-197 -> scope-129
| |
| Project[int][0] - scope-199
@@ -239,7 +233,7 @@ f1: Split - scope-202
| |
| f1: Store(file:///tmp/output/f1:org.apache.pig.builtin.PigStorage) - scope-99
| |
-| f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-210
+| f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-210 -> scope-105
|
|---f1: Limit - scope-169
|
Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Order-1.gld
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Order-1.gld?rev=1733627&r1=1733626&r2=1733627&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Order-1.gld (original)
+++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Order-1.gld Fri Mar 4 18:17:39 2016
@@ -4,70 +4,78 @@
#--------------------------------------------------
# TEZ DAG plan: pig-0_scope-0
#--------------------------------------------------
-Tez vertex scope-11 -> Tez vertex scope-20,Tez vertex scope-30,
-Tez vertex scope-20 -> Tez vertex scope-30,
-Tez vertex scope-30 -> Tez vertex scope-32,
-Tez vertex scope-32
+Tez vertex scope-11 -> Tez vertex scope-26,
+Tez vertex scope-26 -> Tez vertex scope-36,
+Tez vertex scope-36 -> Tez vertex scope-37,
+Tez vertex scope-37
Tez vertex scope-11
# Plan on vertex
-Local Rearrange[tuple]{tuple}(false) - scope-14 -> scope-20
+Local Rearrange[tuple]{tuple}(false) - scope-20 -> scope-26
| |
-| Constant(DummyVal) - scope-13
+| Constant(DummyVal) - scope-19
|
-|---New For Each(false,true)[tuple] - scope-19
+|---New For Each(false,true)[tuple] - scope-25
| |
| Project[int][0] - scope-8
| |
- | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-18
+ | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-24
| |
- | |---Project[tuple][*] - scope-17
+ | |---Project[tuple][*] - scope-23
|
- |---ReservoirSample - scope-16
+ |---ReservoirSample - scope-22
|
- |---b: Local Rearrange[tuple]{int}(false) - scope-12 -> scope-30
+ |---a: New For Each(false,false)[bag] - scope-7
| |
- | Project[int][0] - scope-8
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[int] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
|
- |---a: New For Each(false,false)[bag] - scope-7
- | |
- | Cast[int] - scope-2
- | |
- | |---Project[bytearray][0] - scope-1
- | |
- | Cast[int] - scope-5
- | |
- | |---Project[bytearray][1] - scope-4
- |
- |---a: Load(file:///tmp/input:PigStorage(',')) - scope-0
-Tez vertex scope-20
+ |---a: Load(file:///tmp/input:PigStorage(',')) - scope-0
+Tez vertex scope-26
# Plan on vertex
-POValueOutputTez - scope-29 -> [scope-30]
+POValueOutputTez - scope-35 -> [scope-36]
|
-|---New For Each(false)[tuple] - scope-28
+|---New For Each(false)[tuple] - scope-34
| |
- | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.FindQuantilesTez)[tuple] - scope-27
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.FindQuantilesTez)[tuple] - scope-33
| |
- | |---Project[tuple][*] - scope-26
+ | |---Project[tuple][*] - scope-32
|
- |---New For Each(false,false)[tuple] - scope-25
+ |---New For Each(false,false)[tuple] - scope-31
| |
- | Constant(-1) - scope-24
+ | Constant(-1) - scope-30
| |
- | Project[bag][1] - scope-22
+ | Project[bag][1] - scope-28
|
- |---Package(Packager)[tuple]{bytearray} - scope-21
-Tez vertex scope-30
+ |---Package(Packager)[tuple]{bytearray} - scope-27
+Tez vertex scope-36
# Plan on vertex
-POIdentityInOutTez - scope-31 <- scope-11 -> scope-32
+b: Local Rearrange[tuple]{int}(false) - scope-12 -> scope-37
| |
| Project[int][0] - scope-8
-Tez vertex scope-32
+|
+|---a: New For Each(false,false)[bag] - scope-18
+ | |
+ | Cast[int] - scope-15
+ | |
+ | |---Project[bytearray][0] - scope-14
+ | |
+ | Cast[int] - scope-17
+ | |
+ | |---Project[bytearray][1] - scope-16
+ |
+ |---a: Load(file:///tmp/input:PigStorage(',')) - scope-13
+Tez vertex scope-37
# Plan on vertex
b: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-10
|
-|---New For Each(true)[tuple] - scope-35
+|---New For Each(true)[tuple] - scope-40
| |
- | Project[bag][1] - scope-34
+ | Project[bag][1] - scope-39
|
- |---Package(LitePackager)[tuple]{int} - scope-33
+ |---Package(LitePackager)[tuple]{int} - scope-38
Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SkewJoin-1.gld
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SkewJoin-1.gld?rev=1733627&r1=1733626&r2=1733627&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SkewJoin-1.gld (original)
+++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SkewJoin-1.gld Fri Mar 4 18:17:39 2016
@@ -4,68 +4,76 @@
#--------------------------------------------------
# TEZ DAG plan: pig-0_scope-0
#--------------------------------------------------
-Tez vertex scope-27 -> Tez vertex scope-36,Tez vertex scope-46,
-Tez vertex scope-36 -> Tez vertex scope-28,Tez vertex scope-46,
-Tez vertex scope-46 -> Tez vertex scope-50,
-Tez vertex scope-28 -> Tez vertex scope-50,
-Tez vertex scope-50
+Tez vertex scope-27 -> Tez vertex scope-42,
+Tez vertex scope-42 -> Tez vertex scope-28,Tez vertex scope-52,
+Tez vertex scope-52 -> Tez vertex scope-55,
+Tez vertex scope-28 -> Tez vertex scope-55,
+Tez vertex scope-55
Tez vertex scope-27
# Plan on vertex
-Local Rearrange[tuple]{tuple}(false) - scope-31 -> scope-36
+Local Rearrange[tuple]{tuple}(false) - scope-31 -> scope-42
| |
| Constant(DummyVal) - scope-30
|
-|---New For Each(true,true)[tuple] - scope-35
+|---New For Each(true,true)[tuple] - scope-41
| |
| Project[int][0] - scope-16
| |
- | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-34
+ | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-40
| |
- | |---Project[tuple][*] - scope-33
+ | |---Project[tuple][*] - scope-39
|
|---PoissonSample - scope-32
|
- |---Local Rearrange[tuple]{int}(false) - scope-29 -> scope-46
+ |---a: New For Each(false,false)[bag] - scope-7
| |
- | Project[int][0] - scope-16
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[int] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
|
- |---a: New For Each(false,false)[bag] - scope-7
- | |
- | Cast[int] - scope-2
- | |
- | |---Project[bytearray][0] - scope-1
- | |
- | Cast[int] - scope-5
- | |
- | |---Project[bytearray][1] - scope-4
- |
- |---a: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0
-Tez vertex scope-36
+ |---a: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-42
# Plan on vertex
-POValueOutputTez - scope-45 -> [scope-28, scope-46]
+POValueOutputTez - scope-51 -> [scope-28, scope-52]
|
-|---New For Each(false)[tuple] - scope-44
+|---New For Each(false)[tuple] - scope-50
| |
- | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.PartitionSkewedKeysTez)[tuple] - scope-43
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.PartitionSkewedKeysTez)[tuple] - scope-49
| |
- | |---Project[tuple][*] - scope-42
+ | |---Project[tuple][*] - scope-48
|
- |---New For Each(false,false)[tuple] - scope-41
+ |---New For Each(false,false)[tuple] - scope-47
| |
- | Constant(-1) - scope-40
+ | Constant(-1) - scope-46
| |
- | Project[bag][1] - scope-38
+ | Project[bag][1] - scope-44
|
- |---Package(Packager)[tuple]{bytearray} - scope-37
-Tez vertex scope-46
+ |---Package(Packager)[tuple]{bytearray} - scope-43
+Tez vertex scope-52
# Plan on vertex
-POIdentityInOutTez - scope-47 <- scope-27 -> scope-50
+Local Rearrange[tuple]{int}(false) - scope-29 -> scope-55
| |
| Project[int][0] - scope-16
+|
+|---a: New For Each(false,false)[bag] - scope-38
+ | |
+ | Cast[int] - scope-35
+ | |
+ | |---Project[bytearray][0] - scope-34
+ | |
+ | Cast[int] - scope-37
+ | |
+ | |---Project[bytearray][1] - scope-36
+ |
+ |---a: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-33
Tez vertex scope-28
# Plan on vertex
-Partition Rearrange[tuple]{int}(false) - scope-48 -> scope-50
+Partition Rearrange[tuple]{int}(false) - scope-53 -> scope-55
| |
| Project[int][0] - scope-17
|
@@ -80,7 +88,7 @@ Partition Rearrange[tuple]{int}(false) -
| |---Project[bytearray][1] - scope-12
|
|---b: Load(file:///tmp/input2:org.apache.pig.builtin.PigStorage) - scope-8
-Tez vertex scope-50
+Tez vertex scope-55
# Plan on vertex
d: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-26
|
@@ -92,10 +100,10 @@ d: Store(file:///tmp/output:org.apache.p
| |
| Project[int][3] - scope-23
|
- |---New For Each(true,true)[tuple] - scope-54
+ |---New For Each(true,true)[tuple] - scope-59
| |
- | Project[bag][1] - scope-52
+ | Project[bag][1] - scope-57
| |
- | Project[bag][2] - scope-53
+ | Project[bag][2] - scope-58
|
- |---Package(Packager)[tuple]{int} - scope-51
+ |---Package(Packager)[tuple]{int} - scope-56
Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1-OPTOFF.gld?rev=1733627&r1=1733626&r2=1733627&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1-OPTOFF.gld (original)
+++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1-OPTOFF.gld Fri Mar 4 18:17:39 2016
@@ -2,44 +2,44 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: pig-0_scope-1
+# TEZ DAG plan: pig-0_scope-0
#--------------------------------------------------
-Tez vertex scope-45 -> Tez vertex scope-47,
-Tez vertex scope-46 -> Tez vertex scope-47,
-Tez vertex scope-47
+Tez vertex scope-18 -> Tez vertex scope-20,
+Tez vertex scope-19 -> Tez vertex scope-20,
+Tez vertex scope-20
-Tez vertex scope-45
+Tez vertex scope-18
# Plan on vertex
-POValueOutputTez - scope-49 -> [scope-47]
+POValueOutputTez - scope-22 -> [scope-20]
|
-|---a: New For Each(false,false)[bag] - scope-34
+|---a: New For Each(false,false)[bag] - scope-7
| |
- | Cast[int] - scope-29
+ | Cast[int] - scope-2
| |
- | |---Project[bytearray][0] - scope-28
+ | |---Project[bytearray][0] - scope-1
| |
- | Cast[chararray] - scope-32
+ | Cast[chararray] - scope-5
| |
- | |---Project[bytearray][1] - scope-31
+ | |---Project[bytearray][1] - scope-4
|
- |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-27
-Tez vertex scope-46
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-19
# Plan on vertex
-POValueOutputTez - scope-50 -> [scope-47]
+POValueOutputTez - scope-23 -> [scope-20]
|
-|---c: New For Each(false,false)[bag] - scope-42
+|---c: New For Each(false,false)[bag] - scope-15
| |
- | Cast[int] - scope-37
+ | Cast[int] - scope-10
| |
- | |---Project[bytearray][1] - scope-36
+ | |---Project[bytearray][1] - scope-9
| |
- | Cast[chararray] - scope-40
+ | Cast[chararray] - scope-13
| |
- | |---Project[bytearray][0] - scope-39
+ | |---Project[bytearray][0] - scope-12
|
- |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-35
-Tez vertex scope-47
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8
+Tez vertex scope-20
# Plan on vertex
-c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-44
+c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-17
|
-|---POShuffledValueInputTez - scope-48 <- [scope-45, scope-46]
+|---POShuffledValueInputTez - scope-21 <- [scope-18, scope-19]
Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1.gld
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1.gld?rev=1733627&r1=1733626&r2=1733627&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1.gld (original)
+++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1.gld Fri Mar 4 18:17:39 2016
@@ -10,7 +10,7 @@ Tez vertex group scope-24
Tez vertex scope-18
# Plan on vertex
-c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-25
+c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-25 -> scope-17
|
|---a: New For Each(false,false)[bag] - scope-7
| |
@@ -25,7 +25,7 @@ c: Store(file:///tmp/output:org.apache.p
|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
Tez vertex scope-19
# Plan on vertex
-c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-26
+c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-26 -> scope-17
|
|---c: New For Each(false,false)[bag] - scope-15
| |
Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10-OPTOFF.gld?rev=1733627&r1=1733626&r2=1733627&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10-OPTOFF.gld (original)
+++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10-OPTOFF.gld Fri Mar 4 18:17:39 2016
@@ -4,16 +4,16 @@
#--------------------------------------------------
# TEZ DAG plan: pig-0_scope-0
#--------------------------------------------------
-Tez vertex scope-31 -> Tez vertex scope-33,
-Tez vertex scope-32 -> Tez vertex scope-33,
-Tez vertex scope-33 -> Tez vertex scope-38,
-Tez vertex scope-37 -> Tez vertex scope-38,
-Tez vertex scope-38 -> Tez vertex scope-42,
-Tez vertex scope-42
+Tez vertex scope-37 -> Tez vertex scope-39,
+Tez vertex scope-38 -> Tez vertex scope-39,
+Tez vertex scope-39 -> Tez vertex scope-44,
+Tez vertex scope-43 -> Tez vertex scope-44,
+Tez vertex scope-44 -> Tez vertex scope-53,
+Tez vertex scope-53
-Tez vertex scope-31
+Tez vertex scope-37
# Plan on vertex
-POValueOutputTez - scope-35 -> [scope-33]
+POValueOutputTez - scope-41 -> [scope-39]
|
|---a: New For Each(false,false)[bag] - scope-7
| |
@@ -26,9 +26,9 @@ POValueOutputTez - scope-35 -> [scope-3
| |---Project[bytearray][1] - scope-4
|
|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
-Tez vertex scope-32
+Tez vertex scope-38
# Plan on vertex
-POValueOutputTez - scope-36 -> [scope-33]
+POValueOutputTez - scope-42 -> [scope-39]
|
|---c: New For Each(false,false)[bag] - scope-15
| |
@@ -41,14 +41,14 @@ POValueOutputTez - scope-36 -> [scope-3
| |---Project[bytearray][0] - scope-12
|
|---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8
-Tez vertex scope-33
+Tez vertex scope-39
# Plan on vertex
-POValueOutputTez - scope-40 -> [scope-38]
+POValueOutputTez - scope-46 -> [scope-44]
|
-|---POShuffledValueInputTez - scope-34 <- [scope-31, scope-32]
-Tez vertex scope-37
+|---POShuffledValueInputTez - scope-40 <- [scope-37, scope-38]
+Tez vertex scope-43
# Plan on vertex
-POValueOutputTez - scope-41 -> [scope-38]
+POValueOutputTez - scope-47 -> [scope-44]
|
|---d: New For Each(false,false)[bag] - scope-24
| |
@@ -61,15 +61,19 @@ POValueOutputTez - scope-41 -> [scope-3
| |---Project[bytearray][1] - scope-21
|
|---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-17
-Tez vertex scope-38
+Tez vertex scope-44
# Plan on vertex
-f: Local Rearrange[tuple]{int}(false) - scope-28 -> scope-42
+e: Split - scope-54
+| |
+| e: Store(file:///tmp/output1:org.apache.pig.builtin.PigStorage) - scope-29
| |
-| Project[int][0] - scope-29
+| f: Local Rearrange[tuple]{int}(false) - scope-34 -> scope-53
+| | |
+| | Project[int][0] - scope-35
|
-|---POShuffledValueInputTez - scope-39 <- [scope-33, scope-37]
-Tez vertex scope-42
+|---POShuffledValueInputTez - scope-45 <- [scope-39, scope-43]
+Tez vertex scope-53
# Plan on vertex
-f: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-30
+f: Store(file:///tmp/output2:org.apache.pig.builtin.PigStorage) - scope-36
|
-|---f: Package(Packager)[tuple]{int} - scope-27
+|---f: Package(Packager)[tuple]{int} - scope-33
Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10.gld
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10.gld?rev=1733627&r1=1733626&r2=1733627&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10.gld (original)
+++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-10.gld Fri Mar 4 18:17:39 2016
@@ -4,17 +4,22 @@
#--------------------------------------------------
# TEZ DAG plan: pig-0_scope-0
#--------------------------------------------------
-Tez vertex scope-37 -> Tez vertex group scope-43,
-Tez vertex scope-31 -> Tez vertex group scope-43,
-Tez vertex scope-32 -> Tez vertex group scope-43,
-Tez vertex group scope-43 -> Tez vertex scope-42,
-Tez vertex scope-42
+Tez vertex scope-43 -> Tez vertex group scope-55,Tez vertex group scope-56,
+Tez vertex scope-37 -> Tez vertex group scope-55,Tez vertex group scope-56,
+Tez vertex scope-38 -> Tez vertex group scope-55,Tez vertex group scope-56,
+Tez vertex group scope-56 -> Tez vertex scope-53,
+Tez vertex scope-53
+Tez vertex group scope-55
-Tez vertex scope-37
+Tez vertex scope-43
# Plan on vertex
-f: Local Rearrange[tuple]{int}(false) - scope-46 -> scope-42
+e: Split - scope-61
+| |
+| e: Store(file:///tmp/output1:org.apache.pig.builtin.PigStorage) - scope-62 -> scope-29
| |
-| Project[int][0] - scope-47
+| f: Local Rearrange[tuple]{int}(false) - scope-63 -> scope-53
+| | |
+| | Project[int][0] - scope-64
|
|---d: New For Each(false,false)[bag] - scope-24
| |
@@ -27,11 +32,15 @@ f: Local Rearrange[tuple]{int}(false) -
| |---Project[bytearray][1] - scope-21
|
|---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-17
-Tez vertex scope-31
+Tez vertex scope-37
# Plan on vertex
-f: Local Rearrange[tuple]{int}(false) - scope-49 -> scope-42
+e: Split - scope-66
| |
-| Project[int][0] - scope-50
+| e: Store(file:///tmp/output1:org.apache.pig.builtin.PigStorage) - scope-67 -> scope-29
+| |
+| f: Local Rearrange[tuple]{int}(false) - scope-68 -> scope-53
+| | |
+| | Project[int][0] - scope-69
|
|---a: New For Each(false,false)[bag] - scope-7
| |
@@ -44,11 +53,15 @@ f: Local Rearrange[tuple]{int}(false) -
| |---Project[bytearray][1] - scope-4
|
|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
-Tez vertex scope-32
+Tez vertex scope-38
# Plan on vertex
-f: Local Rearrange[tuple]{int}(false) - scope-51 -> scope-42
+e: Split - scope-70
+| |
+| e: Store(file:///tmp/output1:org.apache.pig.builtin.PigStorage) - scope-71 -> scope-29
| |
-| Project[int][0] - scope-52
+| f: Local Rearrange[tuple]{int}(false) - scope-72 -> scope-53
+| | |
+| | Project[int][0] - scope-73
|
|---c: New For Each(false,false)[bag] - scope-15
| |
@@ -61,10 +74,12 @@ f: Local Rearrange[tuple]{int}(false) -
| |---Project[bytearray][0] - scope-12
|
|---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8
-Tez vertex group scope-43 <- [scope-37, scope-31, scope-32] -> scope-42
+Tez vertex group scope-56 <- [scope-43, scope-37, scope-38] -> scope-53
# No plan on vertex group
-Tez vertex scope-42
+Tez vertex scope-53
# Plan on vertex
-f: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-30
+f: Store(file:///tmp/output2:org.apache.pig.builtin.PigStorage) - scope-36
|
-|---f: Package(Packager)[tuple]{int} - scope-27
+|---f: Package(Packager)[tuple]{int} - scope-33
+Tez vertex group scope-55 <- [scope-43, scope-37, scope-38] -> null
+# No plan on vertex group
Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2-OPTOFF.gld?rev=1733627&r1=1733626&r2=1733627&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2-OPTOFF.gld (original)
+++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2-OPTOFF.gld Fri Mar 4 18:17:39 2016
@@ -4,84 +4,84 @@
#--------------------------------------------------
# TEZ DAG plan: pig-0_scope-1
#--------------------------------------------------
-Tez vertex scope-101 -> Tez vertex scope-103,
-Tez vertex scope-102 -> Tez vertex scope-103,
-Tez vertex scope-103 -> Tez vertex scope-107,
-Tez vertex scope-107
+Tez vertex scope-103 -> Tez vertex scope-105,
+Tez vertex scope-104 -> Tez vertex scope-105,
+Tez vertex scope-105 -> Tez vertex scope-109,
+Tez vertex scope-109
-Tez vertex scope-101
+Tez vertex scope-103
# Plan on vertex
-POValueOutputTez - scope-105 -> [scope-103]
+POValueOutputTez - scope-107 -> [scope-105]
|
-|---a: New For Each(false,false)[bag] - scope-79
+|---a: New For Each(false,false)[bag] - scope-81
| |
- | Cast[int] - scope-74
+ | Cast[int] - scope-76
| |
- | |---Project[bytearray][0] - scope-73
+ | |---Project[bytearray][0] - scope-75
| |
- | Cast[int] - scope-77
+ | Cast[int] - scope-79
| |
- | |---Project[bytearray][1] - scope-76
+ | |---Project[bytearray][1] - scope-78
|
- |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-72
-Tez vertex scope-102
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-74
+Tez vertex scope-104
# Plan on vertex
-POValueOutputTez - scope-106 -> [scope-103]
+POValueOutputTez - scope-108 -> [scope-105]
|
-|---c: New For Each(false,false)[bag] - scope-87
+|---c: New For Each(false,false)[bag] - scope-89
| |
- | Cast[int] - scope-82
+ | Cast[int] - scope-84
| |
- | |---Project[bytearray][1] - scope-81
+ | |---Project[bytearray][1] - scope-83
| |
- | Cast[int] - scope-85
+ | Cast[int] - scope-87
| |
- | |---Project[bytearray][0] - scope-84
+ | |---Project[bytearray][0] - scope-86
|
- |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-80
-Tez vertex scope-103
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-82
+Tez vertex scope-105
# Plan on vertex
-d: Local Rearrange[tuple]{int}(false) - scope-120 -> scope-107
+d: Local Rearrange[tuple]{int}(false) - scope-122 -> scope-109
| |
-| Project[int][0] - scope-122
+| Project[int][0] - scope-124
|
-|---e: New For Each(false,false)[bag] - scope-108
+|---e: New For Each(false,false)[bag] - scope-110
| |
- | Project[int][0] - scope-109
+ | Project[int][0] - scope-111
| |
- | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-110
+ | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-112
| |
- | |---Project[bag][1] - scope-111
+ | |---Project[bag][1] - scope-113
| |
- | |---Project[bag][1] - scope-112
+ | |---Project[bag][1] - scope-114
|
- |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-123
+ |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-125
|
- |---POShuffledValueInputTez - scope-104 <- [scope-101, scope-102]
-Tez vertex scope-107
-# Combine plan on edge <scope-103>
-d: Local Rearrange[tuple]{int}(false) - scope-124 -> scope-107
+ |---POShuffledValueInputTez - scope-106 <- [scope-103, scope-104]
+Tez vertex scope-109
+# Combine plan on edge <scope-105>
+d: Local Rearrange[tuple]{int}(false) - scope-126 -> scope-109
| |
-| Project[int][0] - scope-126
+| Project[int][0] - scope-128
|
-|---e: New For Each(false,false)[bag] - scope-113
+|---e: New For Each(false,false)[bag] - scope-115
| |
- | Project[int][0] - scope-114
+ | Project[int][0] - scope-116
| |
- | POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-115
+ | POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-117
| |
- | |---Project[bag][1] - scope-116
+ | |---Project[bag][1] - scope-118
|
- |---d: Package(CombinerPackager)[tuple]{int} - scope-119
+ |---d: Package(CombinerPackager)[tuple]{int} - scope-121
# Plan on vertex
-e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-100
+e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-102
|
-|---e: New For Each(false,false)[bag] - scope-99
+|---e: New For Each(false,false)[bag] - scope-101
| |
- | Project[int][0] - scope-93
+ | Project[int][0] - scope-95
| |
- | POUserFunc(org.apache.pig.builtin.LongSum$Final)[long] - scope-97
+ | POUserFunc(org.apache.pig.builtin.LongSum$Final)[long] - scope-99
| |
- | |---Project[bag][1] - scope-117
+ | |---Project[bag][1] - scope-119
|
- |---d: Package(CombinerPackager)[tuple]{int} - scope-90
+ |---d: Package(CombinerPackager)[tuple]{int} - scope-92
Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2.gld
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2.gld?rev=1733627&r1=1733626&r2=1733627&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2.gld (original)
+++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-2.gld Fri Mar 4 18:17:39 2016
@@ -40,21 +40,21 @@ d: Local Rearrange[tuple]{int}(false) -
|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
Tez vertex scope-30
# Plan on vertex
-d: Local Rearrange[tuple]{int}(false) - scope-69 -> scope-35
+d: Local Rearrange[tuple]{int}(false) - scope-70 -> scope-35
| |
-| Project[int][0] - scope-70
+| Project[int][0] - scope-71
|
-|---e: New For Each(false,false)[bag] - scope-68
+|---e: New For Each(false,false)[bag] - scope-69
| |
- | Project[int][0] - scope-64
+ | Project[int][0] - scope-65
| |
- | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-65
+ | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-66
| |
- | |---Project[bag][1] - scope-66
+ | |---Project[bag][1] - scope-67
| |
- | |---Project[bag][1] - scope-67
+ | |---Project[bag][1] - scope-68
|
- |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-71
+ |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-72
|
|---c: New For Each(false,false)[bag] - scope-15
| |
Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld?rev=1733627&r1=1733626&r2=1733627&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld (original)
+++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9-OPTOFF.gld Fri Mar 4 18:17:39 2016
@@ -2,88 +2,98 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: pig-0_scope-1
+# TEZ DAG plan: pig-0_scope-0
#--------------------------------------------------
-Tez vertex scope-108 -> Tez vertex scope-109,
-Tez vertex scope-102 -> Tez vertex scope-109,
-Tez vertex scope-109
+Tez vertex scope-45 -> Tez vertex scope-52,Tez vertex scope-57,
+Tez vertex scope-52 -> Tez vertex scope-57,
+Tez vertex scope-56 -> Tez vertex scope-57,
+Tez vertex scope-57
-Tez vertex scope-108
+Tez vertex scope-45
# Plan on vertex
-POValueOutputTez - scope-112 -> [scope-109]
-|
-|---c: New For Each(false,false)[bag] - scope-88
- | |
- | Cast[int] - scope-83
- | |
- | |---Project[bytearray][1] - scope-82
- | |
- | Cast[chararray] - scope-86
- | |
- | |---Project[bytearray][0] - scope-85
- |
- |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-81
-Tez vertex scope-102
-# Plan on vertex
-1-12: Split - scope-119
+1-2: Split - scope-69
| |
-| a2: Store(file:///tmp/output/a2:org.apache.pig.builtin.PigStorage) - scope-76
+| a2: Split - scope-68
+| | |
+| | a2: Store(file:///tmp/output/a2:org.apache.pig.builtin.PigStorage) - scope-17
+| | |
+| | POValueOutputTez - scope-60 -> [scope-57]
| |
-| |---a2: Filter[bag] - scope-71
+| |---a2: Filter[bag] - scope-9
| | |
-| | Not[boolean] - scope-75
+| | Not[boolean] - scope-13
| | |
-| | |---Greater Than[boolean] - scope-74
+| | |---Greater Than[boolean] - scope-12
| | |
-| | |---Project[int][0] - scope-72
+| | |---Project[int][0] - scope-10
| | |
-| | |---Constant(100) - scope-73
-| |
-| POValueOutputTez - scope-111 -> [scope-109]
+| | |---Constant(100) - scope-11
| |
-| |---a1: Filter[bag] - scope-77
-| | |
-| | Greater Than[boolean] - scope-80
-| | |
-| | |---Project[int][0] - scope-78
-| | |
-| | |---Constant(100) - scope-79
+| POValueOutputTez - scope-46 -> [scope-52]
+|
+|---a: New For Each(false,false)[bag] - scope-7
+ | |
+ | Cast[int] - scope-2
+ | |
+ | |---Project[bytearray][0] - scope-1
+ | |
+ | Cast[chararray] - scope-5
+ | |
+ | |---Project[bytearray][1] - scope-4
+ |
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
+Tez vertex scope-52
+# Plan on vertex
+POValueOutputTez - scope-59 -> [scope-57]
+|
+|---a1: Filter[bag] - scope-18
+ | |
+ | Greater Than[boolean] - scope-21
+ | |
+ | |---Project[int][0] - scope-19
+ | |
+ | |---Constant(100) - scope-20
+ |
+ |---POValueInputTez - scope-53 <- scope-45
+Tez vertex scope-56
+# Plan on vertex
+POValueOutputTez - scope-61 -> [scope-57]
|
-|---a: New For Each(false,false)[bag] - scope-69
+|---c: New For Each(false,false)[bag] - scope-31
| |
- | Cast[int] - scope-64
+ | Cast[int] - scope-26
| |
- | |---Project[bytearray][0] - scope-63
+ | |---Project[bytearray][1] - scope-25
| |
- | Cast[chararray] - scope-67
+ | Cast[chararray] - scope-29
| |
- | |---Project[bytearray][1] - scope-66
+ | |---Project[bytearray][0] - scope-28
|
- |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-62
-Tez vertex scope-109
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-24
+Tez vertex scope-57
# Plan on vertex
-1-13: Split - scope-118
+1-3: Split - scope-67
| |
-| d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-95
+| d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-38
| |
-| |---d: Filter[bag] - scope-91
+| |---d: Filter[bag] - scope-34
| | |
-| | Greater Than[boolean] - scope-94
+| | Greater Than[boolean] - scope-37
| | |
-| | |---Project[int][0] - scope-92
+| | |---Project[int][0] - scope-35
| | |
-| | |---Constant(500) - scope-93
+| | |---Constant(500) - scope-36
| |
-| e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-101
+| e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-44
| |
-| |---e: Filter[bag] - scope-96
+| |---e: Filter[bag] - scope-39
| | |
-| | Not[boolean] - scope-100
+| | Not[boolean] - scope-43
| | |
-| | |---Greater Than[boolean] - scope-99
+| | |---Greater Than[boolean] - scope-42
| | |
-| | |---Project[int][0] - scope-97
+| | |---Project[int][0] - scope-40
| | |
-| | |---Constant(500) - scope-98
+| | |---Constant(500) - scope-41
|
-|---POShuffledValueInputTez - scope-110 <- [scope-102, scope-108]
+|---POShuffledValueInputTez - scope-58 <- [scope-45, scope-52, scope-56]
Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld?rev=1733627&r1=1733626&r2=1733627&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld (original)
+++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-9.gld Fri Mar 4 18:17:39 2016
@@ -4,16 +4,42 @@
#--------------------------------------------------
# TEZ DAG plan: pig-0_scope-0
#--------------------------------------------------
-Tez vertex scope-40 -> Tez vertex group scope-58,Tez vertex group scope-59,
-Tez vertex scope-46 -> Tez vertex group scope-58,Tez vertex group scope-59,
-Tez vertex group scope-58
-Tez vertex group scope-59
+Tez vertex scope-45 -> Tez vertex group scope-70,Tez vertex group scope-70,Tez vertex group scope-71,Tez vertex group scope-71,
+Tez vertex scope-56 -> Tez vertex group scope-70,Tez vertex group scope-71,
+Tez vertex group scope-70
+Tez vertex group scope-71
-Tez vertex scope-40
+Tez vertex scope-45
# Plan on vertex
-1-2: Split - scope-57
+1-2: Split - scope-69
| |
-| a2: Store(file:///tmp/output/a2:org.apache.pig.builtin.PigStorage) - scope-14
+| a2: Split - scope-68
+| | |
+| | a2: Store(file:///tmp/output/a2:org.apache.pig.builtin.PigStorage) - scope-17
+| | |
+| | 1-3: Split - scope-72
+| | | |
+| | | d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-77 -> scope-38
+| | | |
+| | | |---d: Filter[bag] - scope-73
+| | | | |
+| | | | Greater Than[boolean] - scope-76
+| | | | |
+| | | | |---Project[int][0] - scope-74
+| | | | |
+| | | | |---Constant(500) - scope-75
+| | | |
+| | | e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-83 -> scope-44
+| | | |
+| | | |---e: Filter[bag] - scope-78
+| | | | |
+| | | | Not[boolean] - scope-82
+| | | | |
+| | | | |---Greater Than[boolean] - scope-81
+| | | | |
+| | | | |---Project[int][0] - scope-79
+| | | | |
+| | | | |---Constant(500) - scope-80
| |
| |---a2: Filter[bag] - scope-9
| | |
@@ -25,37 +51,37 @@ Tez vertex scope-40
| | |
| | |---Constant(100) - scope-11
| |
-| 1-3: Split - scope-60
+| 1-3: Split - scope-84
| | |
-| | d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-33
+| | d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-89 -> scope-38
| | |
-| | |---d: Filter[bag] - scope-29
+| | |---d: Filter[bag] - scope-85
| | | |
-| | | Greater Than[boolean] - scope-32
+| | | Greater Than[boolean] - scope-88
| | | |
-| | | |---Project[int][0] - scope-30
+| | | |---Project[int][0] - scope-86
| | | |
-| | | |---Constant(500) - scope-31
+| | | |---Constant(500) - scope-87
| | |
-| | e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-39
+| | e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-95 -> scope-44
| | |
-| | |---e: Filter[bag] - scope-34
+| | |---e: Filter[bag] - scope-90
| | | |
-| | | Not[boolean] - scope-38
+| | | Not[boolean] - scope-94
| | | |
-| | | |---Greater Than[boolean] - scope-37
+| | | |---Greater Than[boolean] - scope-93
| | | |
-| | | |---Project[int][0] - scope-35
+| | | |---Project[int][0] - scope-91
| | | |
-| | | |---Constant(500) - scope-36
+| | | |---Constant(500) - scope-92
| |
-| |---a1: Filter[bag] - scope-15
+| |---a1: Filter[bag] - scope-18
| | |
-| | Greater Than[boolean] - scope-18
+| | Greater Than[boolean] - scope-21
| | |
-| | |---Project[int][0] - scope-16
+| | |---Project[int][0] - scope-19
| | |
-| | |---Constant(100) - scope-17
+| | |---Constant(100) - scope-20
|
|---a: New For Each(false,false)[bag] - scope-7
| |
@@ -68,44 +94,44 @@ Tez vertex scope-40
| |---Project[bytearray][1] - scope-4
|
|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
-Tez vertex scope-46
+Tez vertex scope-56
# Plan on vertex
-1-3: Split - scope-61
+1-3: Split - scope-96
| |
-| d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-33
+| d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-101 -> scope-38
| |
-| |---d: Filter[bag] - scope-29
+| |---d: Filter[bag] - scope-97
| | |
-| | Greater Than[boolean] - scope-32
+| | Greater Than[boolean] - scope-100
| | |
-| | |---Project[int][0] - scope-30
+| | |---Project[int][0] - scope-98
| | |
-| | |---Constant(500) - scope-31
+| | |---Constant(500) - scope-99
| |
-| e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-39
+| e: Store(file:///tmp/output/e:org.apache.pig.builtin.PigStorage) - scope-107 -> scope-44
| |
-| |---e: Filter[bag] - scope-34
+| |---e: Filter[bag] - scope-102
| | |
-| | Not[boolean] - scope-38
+| | Not[boolean] - scope-106
| | |
-| | |---Greater Than[boolean] - scope-37
+| | |---Greater Than[boolean] - scope-105
| | |
-| | |---Project[int][0] - scope-35
+| | |---Project[int][0] - scope-103
| | |
-| | |---Constant(500) - scope-36
+| | |---Constant(500) - scope-104
|
-|---c: New For Each(false,false)[bag] - scope-26
+|---c: New For Each(false,false)[bag] - scope-31
| |
- | Cast[int] - scope-21
+ | Cast[int] - scope-26
| |
- | |---Project[bytearray][1] - scope-20
+ | |---Project[bytearray][1] - scope-25
| |
- | Cast[chararray] - scope-24
+ | Cast[chararray] - scope-29
| |
- | |---Project[bytearray][0] - scope-23
+ | |---Project[bytearray][0] - scope-28
|
- |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-19
-Tez vertex group scope-58 <- [scope-40, scope-46] -> null
+ |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-24
+Tez vertex group scope-70 <- [scope-45, scope-45, scope-56] -> null
# No plan on vertex group
-Tez vertex group scope-59 <- [scope-40, scope-46] -> null
+Tez vertex group scope-71 <- [scope-45, scope-45, scope-56] -> null
# No plan on vertex group
Modified: pig/branches/spark/test/org/apache/pig/test/pigunit/TestPigTest.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/pigunit/TestPigTest.java?rev=1733627&r1=1733626&r2=1733627&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/pigunit/TestPigTest.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/pigunit/TestPigTest.java Fri Mar 4 18:17:39 2016
@@ -26,6 +26,7 @@ import java.io.PrintWriter;
import java.lang.String;
import java.util.HashMap;
import java.util.Map;
+import java.util.Properties;
import junit.framework.ComparisonFailure;
@@ -38,6 +39,7 @@ import org.apache.pig.pigunit.PigTest;
import org.apache.pig.pigunit.pig.PigServer;
import org.apache.pig.test.Util;
import org.apache.pig.tools.parameters.ParseException;
+import org.apache.pig.impl.util.PropertiesUtil;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
@@ -378,16 +380,13 @@ public class TestPigTest {
@Test
public void testDefaultBootup() throws Exception {
- // Test with properties file
- String pigProps = "pig.properties";
+ // Test with properties object
+ Properties pigProps = PropertiesUtil.loadDefaultProperties();
String bootupPath = "/tmp/.temppigbootup";
- File propertyFile = new File(pigProps);
- PrintWriter out = new PrintWriter(new FileWriter(propertyFile));
- out.println("pig.load.default.statements=" + bootupPath);
- out.close();
+ pigProps.setProperty("pig.load.default.statements", bootupPath);
File bootupFile = new File(bootupPath);
- out = new PrintWriter(new FileWriter(bootupFile));
+ PrintWriter out = new PrintWriter(new FileWriter(bootupFile));
out.println("data = LOAD 'top_queries_input_data.txt' AS (query:CHARARRAY, count:INT);");
out.close();
@@ -420,7 +419,7 @@ public class TestPigTest {
// Create a pigunit.pig.PigServer and Cluster to run this test.
PigServer pig = null;
- pig = new PigServer(Util.getLocalTestMode());
+ pig = new PigServer(Util.getLocalTestMode(), pigProps);
final Cluster cluster = new Cluster(pig.getPigContext());
@@ -434,7 +433,6 @@ public class TestPigTest {
test.assertOutput("queries_limit", output);
- propertyFile.delete();
scriptFile.delete();
bootupFile.delete();
}
Modified: pig/branches/spark/test/org/apache/pig/tez/TestGroupConstParallelTez.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/tez/TestGroupConstParallelTez.java?rev=1733627&r1=1733626&r2=1733627&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/tez/TestGroupConstParallelTez.java (original)
+++ pig/branches/spark/test/org/apache/pig/tez/TestGroupConstParallelTez.java Fri Mar 4 18:17:39 2016
@@ -20,6 +20,9 @@ package org.apache.pig.tez;
import static org.junit.Assert.assertEquals;
+import java.util.HashMap;
+
+import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder;
import org.apache.pig.backend.hadoop.executionengine.tez.plan.TezCompiler;
@@ -68,7 +71,7 @@ public class TestGroupConstParallelTez e
parallelismSetter.visit();
DAG tezDag = getTezDAG(tezPlan, pc);
- TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, null);
+ TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, new HashMap<String, LocalResource>());
dagBuilder.visit();
for (Vertex v : tezDag.getVertices()) {
if (!v.getInputVertices().isEmpty()) {
@@ -88,7 +91,7 @@ public class TestGroupConstParallelTez e
parallelismSetter.visit();
DAG tezDag = getTezDAG(tezPlan, pc);
- TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, null);
+ TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, new HashMap<String, LocalResource>());
dagBuilder.visit();
for (Vertex v : tezDag.getVertices()) {
if (!v.getInputVertices().isEmpty()) {
Modified: pig/branches/spark/test/org/apache/pig/tez/TestJobSubmissionTez.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/tez/TestJobSubmissionTez.java?rev=1733627&r1=1733626&r2=1733627&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/tez/TestJobSubmissionTez.java (original)
+++ pig/branches/spark/test/org/apache/pig/tez/TestJobSubmissionTez.java Fri Mar 4 18:17:39 2016
@@ -20,7 +20,10 @@ package org.apache.pig.tez;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.util.HashMap;
+
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.pig.backend.hadoop.executionengine.JobCreationException;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.backend.hadoop.executionengine.tez.TezDagBuilder;
@@ -62,7 +65,7 @@ public class TestJobSubmissionTez extend
parallelismSetter.visit();
DAG tezDag = getTezDAG(tezPlan, pc);
- TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, null);
+ TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, new HashMap<String, LocalResource>());
try {
dagBuilder.visit();
} catch (VisitorException jce) {
@@ -81,7 +84,7 @@ public class TestJobSubmissionTez extend
parallelismSetter.visit();
DAG tezDag = getTezDAG(tezPlan, pc);
- TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, null);
+ TezDagBuilder dagBuilder = new TezDagBuilder(pc, tezPlan, tezDag, new HashMap<String, LocalResource>());
dagBuilder.visit();
for (Vertex v : tezDag.getVertices()) {
if (!v.getInputVertices().isEmpty()) {
Modified: pig/branches/spark/test/org/apache/pig/tez/TestPigStatsTez.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/tez/TestPigStatsTez.java?rev=1733627&r1=1733626&r2=1733627&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/tez/TestPigStatsTez.java (original)
+++ pig/branches/spark/test/org/apache/pig/tez/TestPigStatsTez.java Fri Mar 4 18:17:39 2016
@@ -20,6 +20,7 @@ package org.apache.pig.tez;
import static org.junit.Assert.assertEquals;
import java.io.File;
+import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.pig.backend.executionengine.ExecJob;
@@ -38,7 +39,7 @@ import org.apache.pig.tools.pigstats.tez
public class TestPigStatsTez extends TestPigStats {
@Override
- public void addSettingsToConf(Configuration conf, String scriptFileName) {
+ public void addSettingsToConf(Configuration conf, String scriptFileName) throws IOException {
TezScriptState ss = TezScriptState.get();
ss.setScript(new File(scriptFileName));
ss.addDAGSettingsToConf(conf);