You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by pr...@apache.org on 2014/11/27 13:50:02 UTC
svn commit: r1642132 [12/14] - in /pig/branches/spark: ./ bin/
contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/
contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/datetime/convert/
contrib/piggybank/java/s...
Modified: pig/branches/spark/test/org/apache/pig/test/TestRegisteredJarVisibility.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestRegisteredJarVisibility.java?rev=1642132&r1=1642131&r2=1642132&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestRegisteredJarVisibility.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestRegisteredJarVisibility.java Thu Nov 27 12:49:54 2014
@@ -133,7 +133,7 @@ public class TestRegisteredJarVisibility
@Test
public void testRegisterJarVisibilityMR() throws IOException {
PigServer pigServer = new PigServer(cluster.getExecType(), cluster.getProperties());
- testRegisteredJarVisibility(pigServer, INPUT_FILE.getName());
+ testRegisteredJarVisibility(pigServer, INPUT_FILE.getAbsolutePath());
}
@Test
@@ -144,7 +144,7 @@ public class TestRegisteredJarVisibility
public void testRegisteredJarVisibility(PigServer pigServer, String inputPath) throws IOException {
String query = "register " + jarFile.getAbsolutePath() + ";\n"
- + "a = load '" + inputPath
+ + "a = load '" + Util.generateURI(inputPath, pigServer.getPigContext())
+ "' using org.apache.pig.test.RegisteredJarVisibilityLoader();\n"
// register again to test classloader consistency
+ "register " + jarFile.getAbsolutePath() + ";\n"
Modified: pig/branches/spark/test/org/apache/pig/test/TestSchema.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestSchema.java?rev=1642132&r1=1642131&r2=1642132&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestSchema.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestSchema.java Thu Nov 27 12:49:54 2014
@@ -20,6 +20,7 @@ package org.apache.pig.test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -123,6 +124,41 @@ public class TestSchema {
}
@Test
+ public void testParsingMapSchemasFromString() throws ParserException {
+ assertNotNull(Utils.getSchemaFromString("b:[(a:int)]"));
+ assertNotNull(Utils.getSchemaFromString("b:[someAlias: (b:int)]"));
+ assertNotNull(Utils.getSchemaFromString("a:map[{bag: (a:int)}]"));
+ assertNotNull(Utils.getSchemaFromString("a:map[someAlias: {bag: (a:int)}]"));
+ assertNotNull(Utils.getSchemaFromString("a:map[chararray]"));
+ assertNotNull(Utils.getSchemaFromString("a:map[someAlias: chararray]"));
+ assertNotNull(Utils.getSchemaFromString("a:map[someAlias: (bar: {bag: (a:int)})]"));
+ }
+
+ @Test
+ public void testMapWithoutAlias() throws FrontendException {
+ List<FieldSchema> innerFields = new ArrayList<FieldSchema>();
+ innerFields.add(new FieldSchema(null, DataType.LONG));
+ List<FieldSchema> fields = new ArrayList<FieldSchema>();
+ fields.add(new FieldSchema("mapAlias", new Schema(innerFields), DataType.MAP));
+
+ Schema inputSchema = new Schema(fields);
+ Schema fromString = Utils.getSchemaFromBagSchemaString(inputSchema.toString());
+ assertTrue(Schema.equals(inputSchema, fromString, false, false));
+ }
+
+ @Test
+ public void testMapWithAlias() throws FrontendException {
+ List<FieldSchema> innerFields = new ArrayList<FieldSchema>();
+ innerFields.add(new FieldSchema("valueAlias", DataType.LONG));
+ List<FieldSchema> fields = new ArrayList<FieldSchema>();
+ fields.add(new FieldSchema("mapAlias", new Schema(innerFields), DataType.MAP));
+
+ Schema inputSchema = new Schema(fields);
+ Schema fromString = Utils.getSchemaFromBagSchemaString(inputSchema.toString());
+ assertTrue(Schema.equals(inputSchema, fromString, false, false));
+ }
+
+ @Test
public void testNormalNestedMerge1() {
// Generate two schemas
@@ -879,10 +915,27 @@ public class TestSchema {
}
}
- @Test(expected=ParserException.class)
- public void testGetStringFromSchemaNegative() throws Exception {
- String schemaString = "a:int b:long"; // A comma is missing between fields
- Utils.getSchemaFromString(schemaString);
- fail("The schema string is invalid, so parsing should fail!");
- }
+ @Test(expected = ParserException.class)
+ public void testGetStringFromSchemaNegative() throws Exception {
+ String schemaString = "a:int b:long"; // A comma is missing between fields
+ Utils.getSchemaFromString(schemaString);
+ fail("The schema string is invalid, so parsing should fail!");
+ }
+
+ @Test
+ public void testGetInitialSchemaStringFromSchema() throws ParserException {
+ String[] schemaStrings = {
+ "my_list:{array:(array_element:(num1:int,num2:int))}",
+ "my_list:{array:(array_element:(num1:int,num2:int),c:chararray)}",
+ "bag:{mytuple3:(mytuple2:(mytuple:(f1:int)))}",
+ "bag:{mytuple:(f1:int)}",
+ "{((num1:int,num2:int))}"
+ };
+ for (String schemaString : schemaStrings) {
+ String s1 = Utils.getSchemaFromString(schemaString).toString();
+ //check if we get back the initial schema string
+ String s2 = s1.substring(1, s1.length() - 1).replaceAll("\\s|bag_0:|tuple_0:", "");
+ assertTrue(schemaString.equals(s2));
+ }
+ }
}
Modified: pig/branches/spark/test/org/apache/pig/test/TestScriptLanguage.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestScriptLanguage.java?rev=1642132&r1=1642131&r2=1642132&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestScriptLanguage.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestScriptLanguage.java Thu Nov 27 12:49:54 2014
@@ -662,7 +662,8 @@ public class TestScriptLanguage {
PigStatsUtil.getEmptyPigStats();
// ExecMode.FILE
- stats = PigRunner.run(new String[] { "-f", scriptFile.getAbsolutePath(), "arg0",
+ stats = PigRunner.run(new String[] { "-x", cluster.getExecType().name().toLowerCase(),
+ "-f", scriptFile.getAbsolutePath(), "arg0",
file1 + "," + file2 }, null);
assertEquals(null, stats.getErrorMessage());
assertFileNotExists(file1, file2);
Modified: pig/branches/spark/test/org/apache/pig/test/TestScriptUDF.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestScriptUDF.java?rev=1642132&r1=1642131&r2=1642132&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestScriptUDF.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestScriptUDF.java Thu Nov 27 12:49:54 2014
@@ -231,7 +231,7 @@ public class TestScriptUDF{
String userenv = Shell.WINDOWS?"USERNAME":"USER";
String[] input = {
userenv,
- "PATH"
+ "JAVA_HOME"
};
Util.createInputFile(cluster, "testPythonBuiltinModuleImport1", input);
Modified: pig/branches/spark/test/org/apache/pig/test/TestStore.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestStore.java?rev=1642132&r1=1642131&r2=1642132&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestStore.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestStore.java Thu Nov 27 12:49:54 2014
@@ -82,7 +82,6 @@ import org.apache.pig.parser.ParserExcep
import org.apache.pig.parser.QueryParserDriver;
import org.apache.pig.test.utils.GenRandomData;
import org.apache.pig.test.utils.TestHelper;
-import org.joda.time.DateTimeZone;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assume;
@@ -116,7 +115,6 @@ public class TestStore {
inputFileName = TESTDIR + "/TestStore-" + new Random().nextLong() + ".txt";
outputFileName = TESTDIR + "/TestStore-output-" + new Random().nextLong() + ".txt";
- DateTimeZone.setDefault(DateTimeZone.forOffsetMillis(DateTimeZone.UTC.getOffset(null)));
}
@After
@@ -355,7 +353,7 @@ public class TestStore {
t.append(flds[9].compareTo("")!=0 ? ps.getLoadCaster().bytesToBoolean(flds[9].getBytes()) : null);
t.append(flds[10].compareTo("")!=0 ? ps.getLoadCaster().bytesToDateTime(flds[10].getBytes()) : null);
t.append(flds[11].compareTo("")!=0 ? ps.getLoadCaster().bytesToCharArray(flds[10].getBytes()) : null);
- assertTrue(TestHelper.tupleEquals(inputTuple, t));
+ assertEquals(inputTuple, t);
}
br.close();
}
@@ -904,7 +902,7 @@ public class TestStore {
}
private void checkStorePath(String orig, String expected, boolean isTmp) throws Exception {
- pc.getProperties().setProperty(PigConfiguration.OPT_MULTIQUERY,""+true);
+ pc.getProperties().setProperty(PigConfiguration.PIG_OPT_MULTIQUERY,""+true);
DataStorage dfs = pc.getDfs();
dfs.setActiveContainer(dfs.asContainer("/tmp"));
Modified: pig/branches/spark/test/org/apache/pig/test/TestStreaming.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestStreaming.java?rev=1642132&r1=1642131&r2=1642132&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestStreaming.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestStreaming.java Thu Nov 27 12:49:54 2014
@@ -113,7 +113,7 @@ public class TestStreaming {
// Pig query to run
pigServer.registerQuery("IP = load '"
- + Util.generateURI(Util.encodeEscape(input.toString()),
+ + Util.generateURI(input.toString(),
pigServer.getPigContext()) + "' using "
+ PigStorage.class.getName() + "(',');");
pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > '3';");
@@ -157,7 +157,7 @@ public class TestStreaming {
}
// Pig query to run
pigServer.registerQuery("IP = load '"
- + Util.generateURI(Util.encodeEscape(input.toString()),
+ + Util.generateURI(input.toString(),
pigServer.getPigContext()) + "' using "
+ PigStorage.class.getName() + "(',');");
pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > '3';");
@@ -200,7 +200,7 @@ public class TestStreaming {
// Pig query to run
pigServer.registerQuery("IP = load '"
- + Util.generateURI(Util.encodeEscape(input.toString()),
+ + Util.generateURI(input.toString(),
pigServer.getPigContext()) + "' using "
+ PigStorage.class.getName() + "(',');");
pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > '3';");
@@ -250,7 +250,7 @@ public class TestStreaming {
// Pig query to run
pigServer.registerQuery("IP = load '"
- + Util.generateURI(Util.encodeEscape(input.toString()),
+ + Util.generateURI(input.toString(),
pigServer.getPigContext()) + "' using "
+ PigStorage.class.getName() + "(',');");
pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > '3';");
@@ -318,7 +318,7 @@ public class TestStreaming {
"output(stdout using " + PigStreaming.class.getName() + "(',')) " +
"stderr();");
pigServer.registerQuery("IP = load '"
- + Util.generateURI(Util.encodeEscape(input.toString()),
+ + Util.generateURI(input.toString(),
pigServer.getPigContext())
+ "' using PigStorage(',');");
pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > 3;");
@@ -388,7 +388,7 @@ public class TestStreaming {
"output(stdout using PS ) " +
"stderr();");
pigServer.registerQuery("IP = load '"
- + Util.generateURI(Util.encodeEscape(input.toString()),
+ + Util.generateURI(input.toString(),
pigServer.getPigContext()) + "' using PigStorage(',');");
pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > 3;");
pigServer.registerQuery("STREAMED_DATA = stream FILTERED_DATA " +
@@ -458,7 +458,7 @@ public class TestStreaming {
"input('bar' using " + PigStreaming.class.getName() + "(',')) " +
"stderr();");
pigServer.registerQuery("IP = load '"
- + Util.generateURI(Util.encodeEscape(input.toString()),
+ + Util.generateURI(input.toString(),
pigServer.getPigContext()) + "' using "
+ PigStorage.class.getName() + "(',');");
pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > 3;");
@@ -520,7 +520,7 @@ public class TestStreaming {
"'bar' using " + PigStreaming.class.getName() + "(',')) " +
"stderr();");
pigServer.registerQuery("IP = load '"
- + Util.generateURI(Util.encodeEscape(input.toString()),
+ + Util.generateURI(input.toString(),
pigServer.getPigContext()) + "' using "
+ PigStorage.class.getName() + "(',');");
pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > 3;");
@@ -583,7 +583,7 @@ public class TestStreaming {
"'bar' using PS) " +
"stderr();");
pigServer.registerQuery("IP = load '"
- + Util.generateURI(Util.encodeEscape(input.toString()),
+ + Util.generateURI(input.toString(),
pigServer.getPigContext()) + "' using PigStorage(',');");
pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > 3;");
pigServer.registerQuery("OP = stream FILTERED_DATA through CMD;");
@@ -644,7 +644,7 @@ public class TestStreaming {
"'foobar' using " + PigStreaming.class.getName() + "(',')) " +
"stderr();");
pigServer.registerQuery("IP = load '"
- + Util.generateURI(Util.encodeEscape(input.toString()),
+ + Util.generateURI(input.toString(),
pigServer.getPigContext()) + "' using "
+ PigStorage.class.getName() + "(',');");
pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > 3;");
@@ -697,7 +697,7 @@ public class TestStreaming {
pigServer.registerQuery("define CMD `" + simpleEchoStreamingCommand +
" | " + simpleEchoStreamingCommand + "`;");
pigServer.registerQuery("IP = load '"
- + Util.generateURI(Util.encodeEscape(input.toString()),
+ + Util.generateURI(input.toString(),
pigServer.getPigContext()) + "' using "
+ PigStorage.class.getName() + "(',');");
if(withTypes[i] == true) {
@@ -738,7 +738,7 @@ public class TestStreaming {
pigServer.registerQuery("define CMD `"+ simpleEchoStreamingCommand +
"` input(stdin using " + PigStreamDump.class.getName() + ");");
pigServer.registerQuery("IP = load '"
- + Util.generateURI(Util.encodeEscape(input.toString()),
+ + Util.generateURI(input.toString(),
pigServer.getPigContext()) + "' using "
+ PigStorage.class.getName() + "(',');");
pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > '3';");
@@ -857,7 +857,7 @@ public class TestStreaming {
"output(stdout using " + PigStreaming.class.getName() + "(',')) " +
"stderr();");
pigServer.registerQuery("IP = load '"
- + Util.generateURI(Util.encodeEscape(input.toString()),
+ + Util.generateURI(input.toString(),
pigServer.getPigContext())
+ "' using PigStorage();");
pigServer.registerQuery("STREAMED_DATA = stream IP through CMD;");
Modified: pig/branches/spark/test/org/apache/pig/test/TestStreamingLocal.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestStreamingLocal.java?rev=1642132&r1=1642131&r2=1642132&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestStreamingLocal.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestStreamingLocal.java Thu Nov 27 12:49:54 2014
@@ -98,7 +98,7 @@ public class TestStreamingLocal {
// Pig query to run
pigServer.registerQuery("IP = load '" +
- Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext()) +
+ Util.generateURI(input.toString(), pigServer.getPigContext()) +
"' using " + PigStorage.class.getName() + "(',');");
pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > '3';");
pigServer.registerQuery("S1 = stream FILTERED_DATA through `" +
@@ -141,7 +141,7 @@ public class TestStreamingLocal {
}
// Pig query to run
pigServer.registerQuery("IP = load '" +
- Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext()) +
+ Util.generateURI(input.toString(), pigServer.getPigContext()) +
"' using " + PigStorage.class.getName() + "(',');");
pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > '3';");
if(withTypes[i] == true) {
@@ -183,7 +183,7 @@ public class TestStreamingLocal {
// Pig query to run
pigServer.registerQuery("IP = load '" +
- Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext()) +
+ Util.generateURI(input.toString(), pigServer.getPigContext()) +
"' using " + PigStorage.class.getName() + "(',');");
pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > '3';");
pigServer.registerQuery("GROUPED_DATA = group FILTERED_DATA by $0;");
@@ -232,7 +232,7 @@ public class TestStreamingLocal {
// Pig query to run
pigServer.registerQuery("IP = load '" +
- Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext()) +
+ Util.generateURI(input.toString(), pigServer.getPigContext()) +
"' using " + PigStorage.class.getName() + "(',');");
pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > '3';");
pigServer.registerQuery("S1 = stream FILTERED_DATA through `" +
@@ -280,7 +280,7 @@ public class TestStreamingLocal {
pigServer.registerQuery("define CMD `" + simpleEchoStreamingCommand +
" | " + simpleEchoStreamingCommand + "`;");
pigServer.registerQuery("IP = load '" +
- Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext()) +
+ Util.generateURI(input.toString(), pigServer.getPigContext()) +
"' using " + PigStorage.class.getName() + "(',');");
if(withTypes[i] == true) {
pigServer.registerQuery("OP = stream IP through CMD as (f0:chararray, f1:int);");
@@ -310,11 +310,11 @@ public class TestStreamingLocal {
expected.set(3, 0);
pigServer.registerQuery("A = load '" +
- Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext()) +
+ Util.generateURI(input.toString(), pigServer.getPigContext()) +
"' using " + PigStorage.class.getName() + "(',') as (a0, a1);");
pigServer.registerQuery("B = stream A through `head -1` as (a0, a1);");
pigServer.registerQuery("C = load '" +
- Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext()) +
+ Util.generateURI(input.toString(), pigServer.getPigContext()) +
"' using " + PigStorage.class.getName() + "(',') as (a0, a1);");
pigServer.registerQuery("D = stream C through `head -1` as (a0, a1);");
pigServer.registerQuery("E = join B by a0, D by a0;");
@@ -359,7 +359,7 @@ public class TestStreamingLocal {
pigServer.registerQuery("define CMD `"+ simpleEchoStreamingCommand +
"` input(stdin);");
pigServer.registerQuery("IP = load '" +
- Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext()) +
+ Util.generateURI(input.toString(), pigServer.getPigContext()) +
"' using " + PigStorage.class.getName() + "(',');");
pigServer.registerQuery("FILTERED_DATA = filter IP by $1 > '3';");
if(withTypes[i] == true) {
Modified: pig/branches/spark/test/org/apache/pig/test/TestStringUDFs.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestStringUDFs.java?rev=1642132&r1=1642131&r2=1642132&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestStringUDFs.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestStringUDFs.java Thu Nov 27 12:49:54 2014
@@ -32,11 +32,13 @@ import org.apache.pig.builtin.REPLACE;
import org.apache.pig.builtin.STARTSWITH;
import org.apache.pig.builtin.ENDSWITH;
import org.apache.pig.builtin.STRSPLIT;
+import org.apache.pig.builtin.STRSPLITTOBAG;
import org.apache.pig.builtin.SUBSTRING;
import org.apache.pig.builtin.TRIM;
import org.apache.pig.builtin.LTRIM;
import org.apache.pig.builtin.RTRIM;
import org.apache.pig.builtin.EqualsIgnoreCase;
+import org.apache.pig.data.DataBag;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.junit.Test;
@@ -227,6 +229,77 @@ public class TestStringUDFs {
}
@Test
+ public void testSplitToBag() throws IOException {
+ STRSPLITTOBAG bagSplit = new STRSPLITTOBAG();
+
+ //test no delims in input
+ Tuple testTuple = Util.buildTuple("1 2 3", "4");
+ DataBag outputBag = bagSplit.exec(testTuple);
+ assertEquals("No of records split should be 1", 1, outputBag.size());
+ assertEquals("Split string should match the input string", "(1 2 3)", outputBag.iterator().next().toString());
+
+ //test default delimiter
+ testTuple = Util.buildTuple("1 2 3");
+ outputBag = bagSplit.exec(testTuple);
+ String[] assertionArray = {"1", "2", "3"};
+ assertEquals("No of record split should be " + assertionArray.length, assertionArray.length, outputBag.size());
+
+ int i = 0;
+ for (Tuple t : outputBag) {
+ assertEquals("Assertion tests on split strings", "(" + assertionArray[i] + ")", t.toString());
+ i++;
+ }
+
+ //test split on specified delimiter
+ testTuple = Util.buildTuple("1:2:3", ":");
+ outputBag = bagSplit.exec(testTuple);
+ assertEquals("No of record split should be " + assertionArray.length, assertionArray.length, outputBag.size());
+ i = 0;
+ for (Tuple t : outputBag) {
+ assertEquals("Assertion tests on split strings", "(" + assertionArray[i] + ")", t.toString());
+ i++;
+ }
+
+ // test limiting results with limit
+ testTuple = Util.buildTuple("1:2:3", ":", 2);
+ outputBag = bagSplit.exec(testTuple);
+ assertionArray = new String[]{"1", "2:3"};
+ assertEquals("No of record split should be " + assertionArray.length, assertionArray.length, outputBag.size());
+ i = 0;
+ for (Tuple t : outputBag) {
+ assertEquals("Matched records in split results with limit", "(" + assertionArray[i] + ")", t.toString());
+ i++;
+ }
+
+ // test trimming of whitespace
+ testTuple = Util.buildTuple("1 2 ");
+ outputBag = bagSplit.exec(testTuple);
+ assertionArray = new String[]{"1", "2"};
+ assertEquals("No of record split should be " + assertionArray.length, assertionArray.length, outputBag.size());
+ i = 0;
+ for (Tuple t : outputBag) {
+ assertEquals("Matched records in split results with trimming of whitespaces", "(" + assertionArray[i] + ")", t.toString());
+ i++;
+ }
+
+ // test forcing null matches with length param
+ testTuple = Util.buildTuple("1:2:::", ":", 10);
+ outputBag = bagSplit.exec(testTuple);
+ assertionArray = new String[]{"1", "2", "", "", ""};
+ assertEquals("No of record split should be " + assertionArray.length, assertionArray.length, outputBag.size());
+ i = 0;
+ for (Tuple t : outputBag) {
+ assertEquals("Matched records in split results with forcing null matched with limit", "(" + assertionArray[i] + ")", t.toString());
+ i++;
+ }
+
+ //test wrong schemas
+ testTuple = Util.buildTuple(1, 2, 3);
+ outputBag = bagSplit.exec(testTuple);
+ assertEquals("Wrong Schema checks", null, outputBag);
+ }
+
+ @Test
public void testStartsWith() throws IOException {
STARTSWITH startsWith = new STARTSWITH();
Tuple testTuple1 = Util.buildTuple("foo", "bar");
Modified: pig/branches/spark/test/org/apache/pig/test/TestTuple.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestTuple.java?rev=1642132&r1=1642131&r2=1642132&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestTuple.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestTuple.java Thu Nov 27 12:49:54 2014
@@ -81,31 +81,31 @@ public class TestTuple {
public void testEmptyTupleSize() {
Tuple t = mTupleFactory.newTuple();
long size = t.getMemorySize();
- assertEquals("tuple size",size, 96);
+ assertEquals("tuple size", 88, size);
}
-
+
@Test
public void testEmptyBagSize() {
DataBag bag = DefaultBagFactory.getInstance().newDefaultBag();
long size = bag.getMemorySize();
- assertEquals("bag size",size, 124);
+ assertEquals("bag size", 124, size);
}
-
+
@Test
// See PIG-1443
public void testTupleSizeWithString() {
Tuple t = Util.createTuple(new String[] {"1234567", "bar"});
long size = t.getMemorySize();
- assertEquals("tuple size",size, 200);
+ assertEquals("tuple size", 192, size);
}
-
+
@Test
public void testTupleSizeWithByteArrays() {
Tuple t = mTupleFactory.newTuple();
t.append(new DataByteArray("1234567"));
t.append(new DataByteArray("bar"));
long size = t.getMemorySize();
- assertEquals("tuple size",size, 168);
+ assertEquals("tuple size", 160, size);
}
@Test
@@ -114,7 +114,7 @@ public class TestTuple {
t.append(new Double(0.1));
t.append(new Double(2000.10001));
long size = t.getMemorySize();
- assertEquals("tuple size",size, 128);
+ assertEquals("tuple size", 120, size);
}
@Test
@@ -123,27 +123,27 @@ public class TestTuple {
t.append(new Float(0.1F));
t.append(new Float(2000.10001F));
long size = t.getMemorySize();
- assertEquals("tuple size",size, 128);
+ assertEquals("tuple size", 120, size);
}
-
+
@Test
public void testTupleSizeWithLongs() {
Tuple t = mTupleFactory.newTuple();
t.append(new Long(100));
t.append(new Long(2000));
long size = t.getMemorySize();
- assertEquals("tuple size",size, 128);
+ assertEquals("tuple size", 120, size);
}
-
+
@Test
public void testTupleSizeWithBooleans() {
Tuple t = mTupleFactory.newTuple();
t.append(new Boolean(true));
t.append(new Boolean(false));
long size = t.getMemorySize();
- assertEquals("tuple size",size, 128);
- }
-
+ assertEquals("tuple size", 120, size);
+ }
+
@Test
public void testTupleIterator() {
Tuple t = mTupleFactory.newTuple();
Modified: pig/branches/spark/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java?rev=1642132&r1=1642131&r2=1642132&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java Thu Nov 27 12:49:54 2014
@@ -3745,7 +3745,7 @@ public class TestTypeCheckingValidatorNe
"[k1#good,k2#morning]" };
PigServer ps = new PigServer(ExecType.LOCAL);
File f = org.apache.pig.test.Util.createInputFile("test", ".txt", input);
- String inputFileName = Util.generateURI(Util.encodeEscape(f.getAbsolutePath()), ps.getPigContext());
+ String inputFileName = Util.generateURI(f.getAbsolutePath(), ps.getPigContext());
// load as bytearray and use as map
String query = "a= load '" + inputFileName + "' as (m);"
+ " b = foreach a generate m#'k1';";
Modified: pig/branches/spark/test/org/apache/pig/test/TestUDF.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestUDF.java?rev=1642132&r1=1642131&r2=1642132&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestUDF.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestUDF.java Thu Nov 27 12:49:54 2014
@@ -20,6 +20,7 @@ package org.apache.pig.test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
import java.io.File;
import java.io.FileWriter;
@@ -319,4 +320,31 @@ public class TestUDF {
return l;
}
}
-}
\ No newline at end of file
+
+ @Test
+ // See PIG-4184
+ public void testUDFNullInput() throws Exception {
+ PigServer pig = new PigServer(ExecType.LOCAL);
+ File inputFile = Util.createInputFile("tmp", "",
+ new String[] {"\t", "2\t3"});
+ pig.registerQuery("a = load '"
+ + Util.generateURI(inputFile.toString(), pig.getPigContext())
+ + "' as (i1:int, i2:int);");
+ pig.registerQuery("b = foreach a generate " + IntToBool.class.getName() + "(i1);");
+
+ Iterator<Tuple> iter = pig.openIterator("b");
+ assertEquals(iter.next().toString(), "(false)");
+ assertEquals(iter.next().toString(), "(true)");
+ assertFalse(iter.hasNext());
+ }
+
+ public static class IntToBool extends EvalFunc<Boolean> {
+ @Override
+ public Boolean exec(Tuple input) throws IOException {
+ if (input == null || input.size() == 0)
+ return null;
+ Integer val = (Integer)input.get(0);
+ return (val == null || val == 0) ? false : true;
+ }
+ }
+}
Modified: pig/branches/spark/test/org/apache/pig/test/TestUnion.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestUnion.java?rev=1642132&r1=1642131&r2=1642132&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestUnion.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestUnion.java Thu Nov 27 12:49:54 2014
@@ -95,7 +95,7 @@ public class TestUnion {
POLoad ld1 = GenPhyOp.topLoadOp();
String curDir = System.getProperty("user.dir");
String inpDir = curDir + File.separatorChar + "test/org/apache/pig/test/data/InputFiles/";
- FileSpec fSpec = new FileSpec(Util.generateURI(Util.encodeEscape(inpDir + "passwd"), pc), new FuncSpec(PigStorage.class.getName() , new String[]{":"}));
+ FileSpec fSpec = new FileSpec(Util.generateURI(inpDir + "passwd", pc), new FuncSpec(PigStorage.class.getName() , new String[]{":"}));
ld1.setLFile(fSpec);
POLoad ld2 = GenPhyOp.topLoadOp();
@@ -290,8 +290,8 @@ public class TestUnion {
PigServer ps = new PigServer(ExecType.LOCAL, new Properties());
//PigStorage and TextLoader have different LoadCasters
- ps.registerQuery("A = load '" + f1.getAbsolutePath() + "' as (a:bytearray);");
- ps.registerQuery("B = load '" + f2.getAbsolutePath() + "' using TextLoader() as (b:bytearray);");
+ ps.registerQuery("A = load '" + Util.encodeEscape(f1.getAbsolutePath()) + "' as (a:bytearray);");
+ ps.registerQuery("B = load '" + Util.encodeEscape(f2.getAbsolutePath()) + "' using TextLoader() as (b:bytearray);");
ps.registerQuery("C = union onschema A,B;");
ps.registerQuery("D = foreach C generate (int)a as a,(chararray)b as b;");
@@ -321,9 +321,9 @@ public class TestUnion {
File f3 = Util.createInputFile("tmp", "i3.txt", new String[] {"1","2","3"});
PigServer ps = new PigServer(ExecType.LOCAL, new Properties());
- ps.registerQuery("A = load '" + f1.getAbsolutePath() + "' as (a:bytearray);"); // Using PigStorage()
- ps.registerQuery("B = load '" + f2.getAbsolutePath() + "' using TextLoader() as (i:bytearray);");
- ps.registerQuery("C = load '" + f3.getAbsolutePath() + "' using TextLoader() as (i:bytearray);");
+ ps.registerQuery("A = load '" + Util.encodeEscape(f1.getAbsolutePath()) + "' as (a:bytearray);"); // Using PigStorage()
+ ps.registerQuery("B = load '" + Util.encodeEscape(f2.getAbsolutePath()) + "' using TextLoader() as (i:bytearray);");
+ ps.registerQuery("C = load '" + Util.encodeEscape(f3.getAbsolutePath()) + "' using TextLoader() as (i:bytearray);");
ps.registerQuery("B2 = join B by i, A by a;"); //{A::a: bytearray,B::i: bytearray}
ps.registerQuery("B3 = foreach B2 generate a, B::i as i;"); //{A::a: bytearray,i: bytearray}
ps.registerQuery("C2 = join C by i, A by a;"); //{A::a: bytearray,C::i: bytearray}
@@ -347,8 +347,8 @@ public class TestUnion {
PigServer ps = new PigServer(ExecType.LOCAL, new Properties());
// PigStorage and PigStorageWithStatistics have the same
// LoadCaster(== Utf8StorageConverter)
- ps.registerQuery("A = load '" + f1.getAbsolutePath() + "' as (a:bytearray, b:bytearray);");
- ps.registerQuery("B = load '" + f1.getAbsolutePath() +
+ ps.registerQuery("A = load '" + Util.encodeEscape(f1.getAbsolutePath()) + "' as (a:bytearray, b:bytearray);");
+ ps.registerQuery("B = load '" + Util.encodeEscape(f1.getAbsolutePath()) +
"' using org.apache.pig.test.PigStorageWithStatistics() as (a:bytearray, b:bytearray);");
ps.registerQuery("C = union onschema A,B;");
ps.registerQuery("D = foreach C generate (int)a as a,(chararray)b as b;");
Modified: pig/branches/spark/test/org/apache/pig/test/TestUnionOnSchema.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/TestUnionOnSchema.java?rev=1642132&r1=1642131&r2=1642132&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/TestUnionOnSchema.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/TestUnionOnSchema.java Thu Nov 27 12:49:54 2014
@@ -492,7 +492,7 @@ public class TestUnionOnSchema {
"(4,5,6,null)",
});
- Util.compareActualAndExpectedResults(list1, expectedRes);
+ Util.checkQueryOutputsAfterSort(list1, expectedRes);
assertEquals(0, list2.size());
}
@@ -852,6 +852,7 @@ public class TestUnionOnSchema {
* Udf that has schema of tuple column with no inner schema
*/
public static class UDFTupleNullSchema extends EvalFunc <Tuple> {
+ @Override
public Tuple exec(Tuple input) {
return input;
}
Modified: pig/branches/spark/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/Util.java?rev=1642132&r1=1642131&r2=1642132&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/Util.java (original)
+++ pig/branches/spark/test/org/apache/pig/test/Util.java Thu Nov 27 12:49:54 2014
@@ -37,6 +37,7 @@ import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.StringReader;
+import java.io.Writer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -53,10 +54,13 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
+import org.apache.log4j.Appender;
import org.apache.log4j.FileAppender;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
+import org.apache.log4j.PatternLayout;
import org.apache.log4j.SimpleLayout;
+import org.apache.log4j.WriterAppender;
import org.apache.pig.ExecType;
import org.apache.pig.ExecTypeProvider;
import org.apache.pig.LoadCaster;
@@ -540,14 +544,10 @@ public class Util {
while(actualResultsIt.hasNext()){
actualResList.add(actualResultsIt.next());
}
-
- compareActualAndExpectedResults(actualResList, expectedResList);
-
+ checkQueryOutputsAfterSort(actualResList, expectedResList);
}
-
-
- static public void compareActualAndExpectedResults(
+ static public void checkQueryOutputsAfterSort(
List<Tuple> actualResList, List<Tuple> expectedResList) {
Collections.sort(actualResList);
Collections.sort(expectedResList);
@@ -611,7 +611,7 @@ public class Util {
}
static private String getMkDirCommandForHadoop2_0(String fileName) {
- if (Util.isHadoop23() || Util.isHadoop2_0()) {
+ if (org.apache.pig.impl.util.Utils.isHadoop23() || org.apache.pig.impl.util.Utils.isHadoop2()) {
Path parentDir = new Path(fileName).getParent();
String mkdirCommand = parentDir.getName().isEmpty() ? "" : "fs -mkdir -p " + parentDir + "\n";
return mkdirCommand;
@@ -630,7 +630,11 @@ public class Util {
static public void copyFromLocalToCluster(MiniGenericCluster cluster,
String localFileName, String fileNameOnCluster) throws IOException {
if(Util.WINDOWS){
- localFileName = localFileName.replace('\\','/');
+ if (!localFileName.contains(":")) {
+ localFileName = localFileName.replace('\\','/');
+ } else {
+ localFileName = localFileName.replace('/','\\');
+ }
fileNameOnCluster = fileNameOnCluster.replace('\\','/');
}
PigServer ps = new PigServer(cluster.getExecType(), cluster.getProperties());
@@ -912,6 +916,38 @@ public class Util {
return executeJavaCommandAndReturnInfo(cmd).exitCode;
}
+ public static class ReadStream implements Runnable {
+ InputStream is;
+ Thread thread;
+ String message = "";
+ public ReadStream(InputStream is) {
+ this.is = is;
+ }
+ public void start () {
+ thread = new Thread (this);
+ thread.start ();
+ }
+ public void run () {
+ try {
+ InputStreamReader isr = new InputStreamReader (is);
+ BufferedReader br = new BufferedReader (isr);
+ while (true) {
+ String s = br.readLine ();
+ if (s == null) break;
+ if (!message.isEmpty()) {
+ message += "\n";
+ }
+ message += s;
+ }
+ is.close ();
+ } catch (Exception ex) {
+ ex.printStackTrace ();
+ }
+ }
+ public String getMessage() {
+ return message;
+ }
+ }
public static ProcessReturnInfo executeJavaCommandAndReturnInfo(String cmd)
throws Exception {
@@ -922,24 +958,17 @@ public class Util {
}
Process cmdProc = Runtime.getRuntime().exec(cmd);
ProcessReturnInfo pri = new ProcessReturnInfo();
- pri.stderrContents = getContents(cmdProc.getErrorStream());
- pri.stdoutContents = getContents(cmdProc.getInputStream());
+ ReadStream stdoutStream = new ReadStream(cmdProc.getInputStream ());
+ ReadStream stderrStream = new ReadStream(cmdProc.getErrorStream ());
+ stdoutStream.start();
+ stderrStream.start();
cmdProc.waitFor();
pri.exitCode = cmdProc.exitValue();
+ pri.stdoutContents = stdoutStream.getMessage();
+ pri.stderrContents = stderrStream.getMessage();
return pri;
}
- private static String getContents(InputStream istr) throws IOException {
- BufferedReader br = new BufferedReader(
- new InputStreamReader(istr));
- String s = "";
- String line;
- while ( (line = br.readLine()) != null) {
- s += line + "\n";
- }
- return s;
-
- }
public static class ProcessReturnInfo {
public int exitCode;
public String stderrContents;
@@ -1232,13 +1261,6 @@ public class Util {
return plan.replaceAll("','','[^']*','scope','true'\\)\\)", "','','','scope','true'))");
}
- public static boolean isHadoop23() {
- String version = org.apache.hadoop.util.VersionInfo.getVersion();
- if (version.matches("\\b0\\.23\\..+\\b"))
- return true;
- return false;
- }
-
public static boolean isHadoop203plus() {
String version = org.apache.hadoop.util.VersionInfo.getVersion();
if (version.matches("\\b0\\.20\\.2\\b"))
@@ -1275,13 +1297,6 @@ public class Util {
assertEquals("Unexpected value found in configs for " + param, expected, conf.getLong(param, -1));
}
- public static boolean isHadoop2_0() {
- String version = org.apache.hadoop.util.VersionInfo.getVersion();
- if (version.matches("\\b2\\.\\d\\..+"))
- return true;
- return false;
- }
-
/**
* Returns a PathFilter that filters out filenames that start with _.
* @return PathFilter
@@ -1357,4 +1372,18 @@ public class Util {
return ExecTypeProvider.fromString("local");
}
}
+
+ public static void createLogAppender(Class clazz, String appenderName, Writer writer) {
+ Logger logger = Logger.getLogger(clazz);
+ WriterAppender writerAppender = new WriterAppender(new PatternLayout("%d [%t] %-5p %c %x - %m%n"), writer);
+ writerAppender.setName(appenderName);
+ logger.addAppender(writerAppender);
+ }
+
+ public static void removeLogAppender(Class clazz, String appenderName) {
+ Logger logger = Logger.getLogger(clazz);
+ Appender appender = logger.getAppender(appenderName);
+ appender.close();
+ logger.removeAppender(appenderName);
+ }
}
Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1-OPTOFF.gld?rev=1642132&r1=1642131&r2=1642132&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1-OPTOFF.gld (original)
+++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1-OPTOFF.gld Thu Nov 27 12:49:54 2014
@@ -2,64 +2,64 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-66
+# TEZ DAG plan: pig-0_scope-1
#--------------------------------------------------
-Tez vertex scope-58 -> Tez vertex scope-60,Tez vertex scope-62,Tez vertex scope-64,
-Tez vertex scope-64
-Tez vertex scope-62
-Tez vertex scope-60
+Tez vertex scope-57 -> Tez vertex scope-59,Tez vertex scope-61,Tez vertex scope-63,
+Tez vertex scope-59
+Tez vertex scope-63
+Tez vertex scope-61
-Tez vertex scope-58
+Tez vertex scope-57
# Plan on vertex
-POValueOutputTez - scope-59 -> [scope-64, scope-62, scope-60]
+POValueOutputTez - scope-58 -> [scope-59, scope-63, scope-61]
|
-|---a: New For Each(false,false)[bag] - scope-41
+|---a: New For Each(false,false)[bag] - scope-40
| |
- | Cast[int] - scope-36
+ | Cast[int] - scope-35
| |
- | |---Project[bytearray][0] - scope-35
+ | |---Project[bytearray][0] - scope-34
| |
- | Cast[int] - scope-39
+ | Cast[int] - scope-38
| |
- | |---Project[bytearray][1] - scope-38
+ | |---Project[bytearray][1] - scope-37
|
- |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-34
-Tez vertex scope-64
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-33
+Tez vertex scope-59
# Plan on vertex
-d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-57
+b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-46
|
-|---d: Filter[bag] - scope-53
+|---b: Filter[bag] - scope-42
| |
- | Greater Than[boolean] - scope-56
+ | Less Than or Equal[boolean] - scope-45
| |
- | |---Project[int][0] - scope-54
+ | |---Project[int][0] - scope-43
| |
- | |---Constant(10) - scope-55
+ | |---Constant(5) - scope-44
|
- |---POValueInputTez - scope-65 <- scope-58
-Tez vertex scope-62
+ |---POValueInputTez - scope-60 <- scope-57
+Tez vertex scope-63
# Plan on vertex
-c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-52
+d: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-56
|
-|---c: Filter[bag] - scope-48
+|---d: Filter[bag] - scope-52
| |
- | Less Than or Equal[boolean] - scope-51
+ | Greater Than[boolean] - scope-55
| |
- | |---Project[int][0] - scope-49
+ | |---Project[int][0] - scope-53
| |
- | |---Constant(10) - scope-50
+ | |---Constant(10) - scope-54
|
- |---POValueInputTez - scope-63 <- scope-58
-Tez vertex scope-60
+ |---POValueInputTez - scope-64 <- scope-57
+Tez vertex scope-61
# Plan on vertex
-b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-47
+c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-51
|
-|---b: Filter[bag] - scope-43
+|---c: Filter[bag] - scope-47
| |
- | Less Than or Equal[boolean] - scope-46
+ | Less Than or Equal[boolean] - scope-50
| |
- | |---Project[int][0] - scope-44
+ | |---Project[int][0] - scope-48
| |
- | |---Constant(5) - scope-45
+ | |---Constant(10) - scope-49
|
- |---POValueInputTez - scope-61 <- scope-58
+ |---POValueInputTez - scope-62 <- scope-57
Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1.gld
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1.gld?rev=1642132&r1=1642131&r2=1642132&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1.gld (original)
+++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-1.gld Thu Nov 27 12:49:54 2014
@@ -2,13 +2,13 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-32
+# TEZ DAG plan: pig-0_scope-0
#--------------------------------------------------
Tez vertex scope-24
Tez vertex scope-24
# Plan on vertex
-1-1: Split - scope-33
+1-1: Split - scope-32
| |
| b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-13
| |
Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld?rev=1642132&r1=1642131&r2=1642132&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld (original)
+++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld Thu Nov 27 12:49:54 2014
@@ -2,305 +2,305 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-393
+# TEZ DAG plan: pig-0_scope-1
#--------------------------------------------------
-Tez vertex scope-318 -> Tez vertex scope-320,Tez vertex scope-331,Tez vertex scope-342,
-Tez vertex scope-320 -> Tez vertex scope-323,Tez vertex scope-336,Tez vertex scope-347,
-Tez vertex scope-347 -> Tez vertex scope-350,Tez vertex scope-376,
-Tez vertex scope-376 -> Tez vertex scope-379,
-Tez vertex scope-379 -> Tez vertex scope-383,Tez vertex scope-387,
-Tez vertex scope-383
-Tez vertex scope-342 -> Tez vertex scope-345,Tez vertex scope-385,
-Tez vertex scope-385 -> Tez vertex group scope-414,
-Tez vertex scope-387 -> Tez vertex group scope-414,
-Tez vertex group scope-414
-Tez vertex scope-331 -> Tez vertex scope-334,Tez vertex scope-339,
-Tez vertex scope-339 -> Tez vertex scope-341,
-Tez vertex scope-341
-Tez vertex scope-323 -> Tez vertex scope-325,
-Tez vertex scope-325 -> Tez vertex scope-327,Tez vertex scope-329,
-Tez vertex scope-327
-Tez vertex scope-345
-Tez vertex scope-329
-Tez vertex scope-350 -> Tez vertex scope-370,Tez vertex scope-360,
-Tez vertex scope-360 -> Tez vertex scope-370,
-Tez vertex scope-370 -> Tez vertex scope-372,
-Tez vertex scope-372
-Tez vertex scope-334 -> Tez vertex scope-338,
-Tez vertex scope-336 -> Tez vertex scope-338,
-Tez vertex scope-338
+Tez vertex scope-317 -> Tez vertex scope-319,Tez vertex scope-330,Tez vertex scope-341,
+Tez vertex scope-341 -> Tez vertex scope-344,Tez vertex scope-384,
+Tez vertex scope-384 -> Tez vertex group scope-412,
+Tez vertex scope-319 -> Tez vertex scope-322,Tez vertex scope-335,Tez vertex scope-346,
+Tez vertex scope-346 -> Tez vertex scope-349,Tez vertex scope-375,
+Tez vertex scope-375 -> Tez vertex scope-378,
+Tez vertex scope-378 -> Tez vertex scope-382,Tez vertex scope-386,
+Tez vertex scope-386 -> Tez vertex group scope-412,
+Tez vertex group scope-412
+Tez vertex scope-382
+Tez vertex scope-330 -> Tez vertex scope-333,Tez vertex scope-338,
+Tez vertex scope-338 -> Tez vertex scope-340,
+Tez vertex scope-340
+Tez vertex scope-322 -> Tez vertex scope-324,
+Tez vertex scope-324 -> Tez vertex scope-326,Tez vertex scope-328,
+Tez vertex scope-326
+Tez vertex scope-344
+Tez vertex scope-328
+Tez vertex scope-349 -> Tez vertex scope-369,Tez vertex scope-359,
+Tez vertex scope-359 -> Tez vertex scope-369,
+Tez vertex scope-369 -> Tez vertex scope-371,
+Tez vertex scope-371
+Tez vertex scope-333 -> Tez vertex scope-337,
+Tez vertex scope-335 -> Tez vertex scope-337,
+Tez vertex scope-337
-Tez vertex scope-318
+Tez vertex scope-317
# Plan on vertex
-POValueOutputTez - scope-319 -> [scope-320, scope-331, scope-342]
+POValueOutputTez - scope-318 -> [scope-341, scope-319, scope-330]
|
-|---a: New For Each(false,false)[bag] - scope-219
+|---a: New For Each(false,false)[bag] - scope-218
| |
- | Cast[int] - scope-214
+ | Cast[int] - scope-213
| |
- | |---Project[bytearray][0] - scope-213
+ | |---Project[bytearray][0] - scope-212
| |
- | Cast[int] - scope-217
+ | Cast[int] - scope-216
| |
- | |---Project[bytearray][1] - scope-216
+ | |---Project[bytearray][1] - scope-215
|
- |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-212
-Tez vertex scope-320
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-211
+Tez vertex scope-341
# Plan on vertex
-POValueOutputTez - scope-322 -> [scope-336, scope-323, scope-347]
+POValueOutputTez - scope-343 -> [scope-384, scope-344]
|
-|---b: Filter[bag] - scope-221
+|---d1: Filter[bag] - scope-284
| |
- | Less Than or Equal[boolean] - scope-224
+ | Equal To[boolean] - scope-287
| |
- | |---Project[int][0] - scope-222
+ | |---Project[int][0] - scope-285
| |
- | |---Constant(5) - scope-223
+ | |---Constant(5) - scope-286
|
- |---POValueInputTez - scope-321 <- scope-318
-Tez vertex scope-347
-# Plan on vertex
-POValueOutputTez - scope-349 -> [scope-376, scope-350]
-|
-|---POValueInputTez - scope-348 <- scope-320
-Tez vertex scope-376
-# Plan on vertex
-POValueOutputTez - scope-378 -> [scope-379]
-|
-|---f1: Limit - scope-307
- |
- |---f: Filter[bag] - scope-303
+ |---d: Filter[bag] - scope-280
| |
- | Greater Than or Equal[boolean] - scope-306
+ | Greater Than[boolean] - scope-283
| |
- | |---Project[int][0] - scope-304
+ | |---Project[int][0] - scope-281
| |
- | |---Constant(3) - scope-305
+ | |---Constant(10) - scope-282
|
- |---POValueInputTez - scope-377 <- scope-347
-Tez vertex scope-379
-# Plan on vertex
-POValueOutputTez - scope-382 -> [scope-387, scope-383]
-|
-|---f1: Limit - scope-381
- |
- |---POValueInputTez - scope-380 <- scope-376
-Tez vertex scope-383
+ |---POValueInputTez - scope-342 <- scope-317
+Tez vertex scope-384
# Plan on vertex
-f1: Store(file:///tmp/output/f1:org.apache.pig.builtin.PigStorage) - scope-311
+f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-413
|
-|---POValueInputTez - scope-384 <- scope-379
-Tez vertex scope-342
+|---POValueInputTez - scope-385 <- scope-341
+Tez vertex scope-319
# Plan on vertex
-POValueOutputTez - scope-344 -> [scope-385, scope-345]
+POValueOutputTez - scope-321 -> [scope-335, scope-322, scope-346]
|
-|---d1: Filter[bag] - scope-285
+|---b: Filter[bag] - scope-220
| |
- | Equal To[boolean] - scope-288
+ | Less Than or Equal[boolean] - scope-223
| |
- | |---Project[int][0] - scope-286
+ | |---Project[int][0] - scope-221
| |
- | |---Constant(5) - scope-287
+ | |---Constant(5) - scope-222
|
- |---d: Filter[bag] - scope-281
+ |---POValueInputTez - scope-320 <- scope-317
+Tez vertex scope-346
+# Plan on vertex
+POValueOutputTez - scope-348 -> [scope-375, scope-349]
+|
+|---POValueInputTez - scope-347 <- scope-319
+Tez vertex scope-375
+# Plan on vertex
+POValueOutputTez - scope-377 -> [scope-378]
+|
+|---f1: Limit - scope-306
+ |
+ |---f: Filter[bag] - scope-302
| |
- | Greater Than[boolean] - scope-284
+ | Greater Than or Equal[boolean] - scope-305
| |
- | |---Project[int][0] - scope-282
+ | |---Project[int][0] - scope-303
| |
- | |---Constant(10) - scope-283
+ | |---Constant(3) - scope-304
|
- |---POValueInputTez - scope-343 <- scope-318
-Tez vertex scope-385
+ |---POValueInputTez - scope-376 <- scope-346
+Tez vertex scope-378
# Plan on vertex
-f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-415
+POValueOutputTez - scope-381 -> [scope-386, scope-382]
|
-|---POValueInputTez - scope-386 <- scope-342
-Tez vertex scope-387
+|---f1: Limit - scope-380
+ |
+ |---POValueInputTez - scope-379 <- scope-375
+Tez vertex scope-386
# Plan on vertex
-f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-416
+f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-414
|
-|---POValueInputTez - scope-388 <- scope-379
-Tez vertex group scope-414 <- [scope-385, scope-387] -> null
+|---POValueInputTez - scope-387 <- scope-378
+Tez vertex group scope-412 <- [scope-384, scope-386] -> null
# No plan on vertex group
-Tez vertex scope-331
+Tez vertex scope-382
# Plan on vertex
-POValueOutputTez - scope-333 -> [scope-339, scope-334]
+f1: Store(file:///tmp/output/f1:org.apache.pig.builtin.PigStorage) - scope-310
|
-|---c: Filter[bag] - scope-246
+|---POValueInputTez - scope-383 <- scope-378
+Tez vertex scope-330
+# Plan on vertex
+POValueOutputTez - scope-332 -> [scope-338, scope-333]
+|
+|---c: Filter[bag] - scope-245
| |
- | Less Than or Equal[boolean] - scope-249
+ | Less Than or Equal[boolean] - scope-248
| |
- | |---Project[int][0] - scope-247
+ | |---Project[int][0] - scope-246
| |
- | |---Constant(10) - scope-248
+ | |---Constant(10) - scope-247
|
- |---POValueInputTez - scope-332 <- scope-318
-Tez vertex scope-339
+ |---POValueInputTez - scope-331 <- scope-317
+Tez vertex scope-338
# Plan on vertex
-c2: Local Rearrange[tuple]{int}(false) - scope-406 -> scope-341
+c2: Local Rearrange[tuple]{int}(false) - scope-404 -> scope-340
| |
-| Project[int][0] - scope-408
+| Project[int][0] - scope-406
|
-|---c3: New For Each(false,false)[bag] - scope-394
+|---c3: New For Each(false,false)[bag] - scope-392
| |
- | Project[int][0] - scope-395
+ | Project[int][0] - scope-393
| |
- | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-396
+ | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-394
| |
- | |---Project[bag][0] - scope-397
+ | |---Project[bag][0] - scope-395
| |
- | |---Project[bag][1] - scope-398
+ | |---Project[bag][1] - scope-396
|
- |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-409
+ |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-407
|
- |---POValueInputTez - scope-340 <- scope-331
-Tez vertex scope-341
-# Combine plan on edge <scope-339>
-c2: Local Rearrange[tuple]{int}(false) - scope-410 -> scope-341
+ |---POValueInputTez - scope-339 <- scope-330
+Tez vertex scope-340
+# Combine plan on edge <scope-338>
+c2: Local Rearrange[tuple]{int}(false) - scope-408 -> scope-340
| |
-| Project[int][0] - scope-412
+| Project[int][0] - scope-410
|
-|---c3: New For Each(false,false)[bag] - scope-399
+|---c3: New For Each(false,false)[bag] - scope-397
| |
- | Project[int][0] - scope-400
+ | Project[int][0] - scope-398
| |
- | POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-401
+ | POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-399
| |
- | |---Project[bag][1] - scope-402
+ | |---Project[bag][1] - scope-400
|
- |---c2: Package(CombinerPackager)[tuple]{int} - scope-405
+ |---c2: Package(CombinerPackager)[tuple]{int} - scope-403
# Plan on vertex
-c3: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-280
+c3: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-279
|
-|---c3: New For Each(false,false)[bag] - scope-279
+|---c3: New For Each(false,false)[bag] - scope-278
| |
- | Project[int][0] - scope-273
+ | Project[int][0] - scope-272
| |
- | POUserFunc(org.apache.pig.builtin.LongSum$Final)[long] - scope-277
+ | POUserFunc(org.apache.pig.builtin.LongSum$Final)[long] - scope-276
| |
- | |---Project[bag][1] - scope-403
+ | |---Project[bag][1] - scope-401
|
- |---c2: Package(CombinerPackager)[tuple]{int} - scope-270
-Tez vertex scope-323
+ |---c2: Package(CombinerPackager)[tuple]{int} - scope-269
+Tez vertex scope-322
# Plan on vertex
-b1: Local Rearrange[tuple]{int}(false) - scope-230 -> scope-325
+b1: Local Rearrange[tuple]{int}(false) - scope-229 -> scope-324
| |
-| Project[int][0] - scope-231
+| Project[int][0] - scope-230
|
-|---POValueInputTez - scope-324 <- scope-320
-Tez vertex scope-325
+|---POValueInputTez - scope-323 <- scope-319
+Tez vertex scope-324
# Plan on vertex
-POValueOutputTez - scope-326 -> [scope-329, scope-327]
+POValueOutputTez - scope-325 -> [scope-328, scope-326]
|
-|---b1: Package(Packager)[tuple]{int} - scope-229
-Tez vertex scope-327
+|---b1: Package(Packager)[tuple]{int} - scope-228
+Tez vertex scope-326
# Plan on vertex
-b1: Store(file:///tmp/output/b1:org.apache.pig.builtin.PigStorage) - scope-235
+b1: Store(file:///tmp/output/b1:org.apache.pig.builtin.PigStorage) - scope-234
|
-|---POValueInputTez - scope-328 <- scope-325
-Tez vertex scope-345
+|---POValueInputTez - scope-327 <- scope-324
+Tez vertex scope-344
# Plan on vertex
-d1: Store(file:///tmp/output/d1:org.apache.pig.builtin.PigStorage) - scope-292
+d1: Store(file:///tmp/output/d1:org.apache.pig.builtin.PigStorage) - scope-291
|
-|---POValueInputTez - scope-346 <- scope-342
-Tez vertex scope-329
+|---POValueInputTez - scope-345 <- scope-341
+Tez vertex scope-328
# Plan on vertex
-b2: Store(file:///tmp/output/b2:org.apache.pig.builtin.PigStorage) - scope-245
+b2: Store(file:///tmp/output/b2:org.apache.pig.builtin.PigStorage) - scope-244
|
-|---b2: New For Each(false,false)[bag] - scope-244
+|---b2: New For Each(false,false)[bag] - scope-243
| |
- | Project[int][0] - scope-238
+ | Project[int][0] - scope-237
| |
- | POUserFunc(org.apache.pig.builtin.LongSum)[long] - scope-242
+ | POUserFunc(org.apache.pig.builtin.LongSum)[long] - scope-241
| |
- | |---Project[bag][0] - scope-241
+ | |---Project[bag][0] - scope-240
| |
- | |---Project[bag][1] - scope-240
+ | |---Project[bag][1] - scope-239
|
- |---POValueInputTez - scope-330 <- scope-325
-Tez vertex scope-350
+ |---POValueInputTez - scope-329 <- scope-324
+Tez vertex scope-349
# Plan on vertex
-Local Rearrange[tuple]{tuple}(false) - scope-354 -> scope-360
+Local Rearrange[tuple]{tuple}(false) - scope-353 -> scope-359
| |
-| Constant(DummyVal) - scope-353
+| Constant(DummyVal) - scope-352
|
-|---New For Each(false,true)[tuple] - scope-359
+|---New For Each(false,true)[tuple] - scope-358
| |
- | Project[int][0] - scope-300
+ | Project[int][0] - scope-299
| |
- | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-358
+ | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-357
| |
- | |---Project[tuple][*] - scope-357
+ | |---Project[tuple][*] - scope-356
|
- |---ReservoirSample - scope-356
+ |---ReservoirSample - scope-355
|
- |---e1: Local Rearrange[tuple]{int}(false) - scope-352 -> scope-370
+ |---e1: Local Rearrange[tuple]{int}(false) - scope-351 -> scope-369
| |
- | Project[int][0] - scope-300
+ | Project[int][0] - scope-299
|
- |---e: Filter[bag] - scope-296
+ |---e: Filter[bag] - scope-295
| |
- | Less Than[boolean] - scope-299
+ | Less Than[boolean] - scope-298
| |
- | |---Project[int][0] - scope-297
+ | |---Project[int][0] - scope-296
| |
- | |---Constant(3) - scope-298
+ | |---Constant(3) - scope-297
|
- |---POValueInputTez - scope-351 <- scope-347
-Tez vertex scope-360
+ |---POValueInputTez - scope-350 <- scope-346
+Tez vertex scope-359
# Plan on vertex
-POValueOutputTez - scope-369 -> [scope-370]
+POValueOutputTez - scope-368 -> [scope-369]
|
-|---New For Each(false)[tuple] - scope-368
+|---New For Each(false)[tuple] - scope-367
| |
- | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-367
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.FindQuantilesTez)[tuple] - scope-366
| |
- | |---Project[tuple][*] - scope-366
+ | |---Project[tuple][*] - scope-365
|
- |---New For Each(false,false)[tuple] - scope-365
+ |---New For Each(false,false)[tuple] - scope-364
| |
- | Constant(-1) - scope-364
+ | Constant(-1) - scope-363
| |
- | Project[bag][1] - scope-362
+ | Project[bag][1] - scope-361
|
- |---Package(Packager)[tuple]{bytearray} - scope-361
-Tez vertex scope-370
+ |---Package(Packager)[tuple]{bytearray} - scope-360
+Tez vertex scope-369
# Plan on vertex
-POIdentityInOutTez - scope-371 <- scope-350 -> scope-372
+POIdentityInOutTez - scope-370 <- scope-349 -> scope-371
| |
-| Project[int][0] - scope-300
-Tez vertex scope-372
+| Project[int][0] - scope-299
+Tez vertex scope-371
# Plan on vertex
-e1: Store(file:///tmp/output/e1:org.apache.pig.builtin.PigStorage) - scope-302
+e1: Store(file:///tmp/output/e1:org.apache.pig.builtin.PigStorage) - scope-301
|
-|---New For Each(true)[tuple] - scope-375
+|---New For Each(true)[tuple] - scope-374
| |
- | Project[bag][1] - scope-374
+ | Project[bag][1] - scope-373
|
- |---Package(LitePackager)[tuple]{int} - scope-373
-Tez vertex scope-334
+ |---Package(LitePackager)[tuple]{int} - scope-372
+Tez vertex scope-333
# Plan on vertex
-c1: Local Rearrange[tuple]{int}(false) - scope-259 -> scope-338
+c1: Local Rearrange[tuple]{int}(false) - scope-258 -> scope-337
| |
-| Project[int][0] - scope-260
+| Project[int][0] - scope-259
|
-|---POValueInputTez - scope-335 <- scope-331
-Tez vertex scope-336
+|---POValueInputTez - scope-334 <- scope-330
+Tez vertex scope-335
# Plan on vertex
-c1: Local Rearrange[tuple]{int}(false) - scope-261 -> scope-338
+c1: Local Rearrange[tuple]{int}(false) - scope-260 -> scope-337
| |
-| Project[int][0] - scope-262
+| Project[int][0] - scope-261
|
-|---POValueInputTez - scope-337 <- scope-320
-Tez vertex scope-338
+|---POValueInputTez - scope-336 <- scope-319
+Tez vertex scope-337
# Plan on vertex
-c1: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-266
+c1: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-265
|
-|---c1: New For Each(true,true)[tuple] - scope-265
+|---c1: New For Each(true,true)[tuple] - scope-264
| |
- | Project[bag][1] - scope-263
+ | Project[bag][1] - scope-262
| |
- | Project[bag][2] - scope-264
+ | Project[bag][2] - scope-263
|
- |---c1: Package(Packager)[tuple]{int} - scope-258
+ |---c1: Package(Packager)[tuple]{int} - scope-257
Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld?rev=1642132&r1=1642131&r2=1642132&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld (original)
+++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld Thu Nov 27 12:49:54 2014
@@ -2,11 +2,11 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-181
+# TEZ DAG plan: pig-0_scope-0
#--------------------------------------------------
-Tez vertex scope-106 -> Tez vertex scope-119,Tez vertex scope-113,Tez vertex scope-126,Tez vertex scope-158,Tez vertex scope-148,Tez vertex scope-167,Tez vertex group scope-209,
-Tez vertex scope-167 -> Tez vertex group scope-209,
-Tez vertex group scope-209
+Tez vertex scope-106 -> Tez vertex scope-119,Tez vertex scope-113,Tez vertex scope-126,Tez vertex scope-158,Tez vertex scope-148,Tez vertex scope-167,Tez vertex group scope-208,
+Tez vertex scope-167 -> Tez vertex group scope-208,
+Tez vertex group scope-208
Tez vertex scope-113
Tez vertex scope-148 -> Tez vertex scope-158,
Tez vertex scope-158 -> Tez vertex scope-160,
@@ -17,9 +17,9 @@ Tez vertex scope-129
Tez vertex scope-106
# Plan on vertex
-1-1: Split - scope-208
+1-1: Split - scope-207
| |
-| b: Split - scope-205
+| b: Split - scope-204
| | |
| | b1: Local Rearrange[tuple]{int}(false) - scope-18 -> scope-113
| | | |
@@ -29,7 +29,7 @@ Tez vertex scope-106
| | | |
| | | Project[int][0] - scope-50
| | |
-| | 1-2: Split - scope-204
+| | 1-2: Split - scope-203
| | | |
| | | Local Rearrange[tuple]{tuple}(false) - scope-142 -> scope-148
| | | | |
@@ -77,11 +77,11 @@ Tez vertex scope-106
| | |
| | |---Constant(5) - scope-11
| |
-| d1: Split - scope-207
+| d1: Split - scope-206
| | |
| | d1: Store(file:///tmp/output/d1:org.apache.pig.builtin.PigStorage) - scope-80
| | |
-| | f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-210
+| | f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-209
| |
| |---d1: Filter[bag] - scope-73
| | |
@@ -114,20 +114,20 @@ Tez vertex scope-106
|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
Tez vertex scope-167
# Plan on vertex
-f1: Split - scope-203
+f1: Split - scope-202
| |
| f1: Store(file:///tmp/output/f1:org.apache.pig.builtin.PigStorage) - scope-99
| |
-| f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-211
+| f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-210
|
|---f1: Limit - scope-169
|
|---POValueInputTez - scope-168 <- scope-106
-Tez vertex group scope-209 <- [scope-106, scope-167] -> null
+Tez vertex group scope-208 <- [scope-106, scope-167] -> null
# No plan on vertex group
Tez vertex scope-113
# Plan on vertex
-b1: Split - scope-202
+b1: Split - scope-201
| |
| b1: Store(file:///tmp/output/b1:org.apache.pig.builtin.PigStorage) - scope-23
| |
@@ -150,7 +150,7 @@ POValueOutputTez - scope-157 -> [scope-
|
|---New For Each(false)[tuple] - scope-156
| |
- | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-155
+ | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.FindQuantilesTez)[tuple] - scope-155
| |
| |---Project[tuple][*] - scope-154
|
@@ -177,27 +177,27 @@ e1: Store(file:///tmp/output/e1:org.apac
|---Package(LitePackager)[tuple]{int} - scope-161
Tez vertex scope-119
# Plan on vertex
-c: Split - scope-206
+c: Split - scope-205
| |
| c1: Local Rearrange[tuple]{int}(false) - scope-47 -> scope-126
| | |
| | Project[int][0] - scope-48
| |
-| c2: Local Rearrange[tuple]{int}(false) - scope-194 -> scope-129
+| c2: Local Rearrange[tuple]{int}(false) - scope-193 -> scope-129
| | |
-| | Project[int][0] - scope-196
+| | Project[int][0] - scope-195
| |
-| |---c3: New For Each(false,false)[bag] - scope-182
+| |---c3: New For Each(false,false)[bag] - scope-181
| | |
-| | Project[int][0] - scope-183
+| | Project[int][0] - scope-182
| | |
-| | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-184
+| | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-183
| | |
-| | |---Project[bag][0] - scope-185
+| | |---Project[bag][0] - scope-184
| | |
-| | |---Project[bag][1] - scope-186
+| | |---Project[bag][1] - scope-185
| |
-| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-197
+| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-196
|
|---c: Filter[bag] - scope-34
| |
@@ -221,19 +221,19 @@ c1: Store(file:///tmp/output/c1:org.apac
|---c1: Package(Packager)[tuple]{int} - scope-46
Tez vertex scope-129
# Combine plan on edge <scope-119>
-c2: Local Rearrange[tuple]{int}(false) - scope-198 -> scope-129
+c2: Local Rearrange[tuple]{int}(false) - scope-197 -> scope-129
| |
-| Project[int][0] - scope-200
+| Project[int][0] - scope-199
|
-|---c3: New For Each(false,false)[bag] - scope-187
+|---c3: New For Each(false,false)[bag] - scope-186
| |
- | Project[int][0] - scope-188
+ | Project[int][0] - scope-187
| |
- | POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-189
+ | POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-188
| |
- | |---Project[bag][1] - scope-190
+ | |---Project[bag][1] - scope-189
|
- |---c2: Package(CombinerPackager)[tuple]{int} - scope-193
+ |---c2: Package(CombinerPackager)[tuple]{int} - scope-192
# Plan on vertex
c3: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-68
|
@@ -243,6 +243,6 @@ c3: Store(file:///tmp/output/c1:org.apac
| |
| POUserFunc(org.apache.pig.builtin.LongSum$Final)[long] - scope-65
| |
- | |---Project[bag][1] - scope-191
+ | |---Project[bag][1] - scope-190
|
|---c2: Package(CombinerPackager)[tuple]{int} - scope-58
Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3-OPTOFF.gld?rev=1642132&r1=1642131&r2=1642132&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3-OPTOFF.gld (original)
+++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3-OPTOFF.gld Thu Nov 27 12:49:54 2014
@@ -2,118 +2,118 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-134
+# TEZ DAG plan: pig-0_scope-1
#--------------------------------------------------
-Tez vertex scope-126 -> Tez vertex scope-128,Tez vertex scope-131,
-Tez vertex scope-131 -> Tez vertex scope-133,
-Tez vertex scope-133
-Tez vertex scope-128 -> Tez vertex scope-130,
-Tez vertex scope-130
+Tez vertex scope-125 -> Tez vertex scope-127,Tez vertex scope-130,
+Tez vertex scope-127 -> Tez vertex scope-129,
+Tez vertex scope-129
+Tez vertex scope-130 -> Tez vertex scope-132,
+Tez vertex scope-132
-Tez vertex scope-126
+Tez vertex scope-125
# Plan on vertex
-POValueOutputTez - scope-127 -> [scope-128, scope-131]
+POValueOutputTez - scope-126 -> [scope-127, scope-130]
|
-|---a: New For Each(false,false)[bag] - scope-95
+|---a: New For Each(false,false)[bag] - scope-94
| |
- | Cast[int] - scope-90
+ | Cast[int] - scope-89
| |
- | |---Project[bytearray][0] - scope-89
+ | |---Project[bytearray][0] - scope-88
| |
- | Cast[int] - scope-93
+ | Cast[int] - scope-92
| |
- | |---Project[bytearray][1] - scope-92
+ | |---Project[bytearray][1] - scope-91
|
- |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-88
-Tez vertex scope-131
+ |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-87
+Tez vertex scope-127
# Plan on vertex
-c: Local Rearrange[tuple]{tuple}(false) - scope-166 -> scope-133
+b: Local Rearrange[tuple]{int}(false) - scope-145 -> scope-129
| |
-| Project[tuple][0] - scope-169
+| Project[int][0] - scope-147
|
-|---c: New For Each(false,false)[bag] - scope-154
+|---b: New For Each(false,false)[bag] - scope-133
| |
- | Project[tuple][0] - scope-155
+ | Project[int][0] - scope-134
| |
- | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-156
+ | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-135
| |
- | |---Project[bag][1] - scope-157
+ | |---Project[bag][0] - scope-136
| |
- | |---Project[bag][1] - scope-158
+ | |---Project[bag][1] - scope-137
|
- |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-170
+ |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-148
|
- |---POValueInputTez - scope-132 <- scope-126
-Tez vertex scope-133
-# Combine plan on edge <scope-131>
-c: Local Rearrange[tuple]{tuple}(false) - scope-171 -> scope-133
+ |---POValueInputTez - scope-128 <- scope-125
+Tez vertex scope-129
+# Combine plan on edge <scope-127>
+b: Local Rearrange[tuple]{int}(false) - scope-149 -> scope-129
| |
-| Project[tuple][0] - scope-174
+| Project[int][0] - scope-151
|
-|---c: New For Each(false,false)[bag] - scope-159
+|---b: New For Each(false,false)[bag] - scope-138
| |
- | Project[tuple][0] - scope-160
+ | Project[int][0] - scope-139
| |
- | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-161
+ | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-140
| |
- | |---Project[bag][1] - scope-162
+ | |---Project[bag][1] - scope-141
|
- |---c: Package(CombinerPackager)[tuple]{tuple} - scope-165
+ |---b: Package(CombinerPackager)[tuple]{int} - scope-144
# Plan on vertex
-c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-125
+b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-109
|
-|---c: New For Each(false,false)[bag] - scope-124
+|---b: New For Each(false,false)[bag] - scope-108
| |
- | Project[tuple][0] - scope-118
+ | Project[int][0] - scope-102
| |
- | POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-122
+ | POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-106
| |
- | |---Project[bag][1] - scope-163
+ | |---Project[bag][1] - scope-142
|
- |---c: Package(CombinerPackager)[tuple]{tuple} - scope-114
-Tez vertex scope-128
+ |---b: Package(CombinerPackager)[tuple]{int} - scope-99
+Tez vertex scope-130
# Plan on vertex
-b: Local Rearrange[tuple]{int}(false) - scope-147 -> scope-130
+c: Local Rearrange[tuple]{tuple}(false) - scope-164 -> scope-132
| |
-| Project[int][0] - scope-149
+| Project[tuple][0] - scope-167
|
-|---b: New For Each(false,false)[bag] - scope-135
+|---c: New For Each(false,false)[bag] - scope-152
| |
- | Project[int][0] - scope-136
+ | Project[tuple][0] - scope-153
| |
- | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-137
+ | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-154
| |
- | |---Project[bag][0] - scope-138
+ | |---Project[bag][1] - scope-155
| |
- | |---Project[bag][1] - scope-139
+ | |---Project[bag][1] - scope-156
|
- |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-150
+ |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-168
|
- |---POValueInputTez - scope-129 <- scope-126
-Tez vertex scope-130
-# Combine plan on edge <scope-128>
-b: Local Rearrange[tuple]{int}(false) - scope-151 -> scope-130
+ |---POValueInputTez - scope-131 <- scope-125
+Tez vertex scope-132
+# Combine plan on edge <scope-130>
+c: Local Rearrange[tuple]{tuple}(false) - scope-169 -> scope-132
| |
-| Project[int][0] - scope-153
+| Project[tuple][0] - scope-172
|
-|---b: New For Each(false,false)[bag] - scope-140
+|---c: New For Each(false,false)[bag] - scope-157
| |
- | Project[int][0] - scope-141
+ | Project[tuple][0] - scope-158
| |
- | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-142
+ | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-159
| |
- | |---Project[bag][1] - scope-143
+ | |---Project[bag][1] - scope-160
|
- |---b: Package(CombinerPackager)[tuple]{int} - scope-146
+ |---c: Package(CombinerPackager)[tuple]{tuple} - scope-163
# Plan on vertex
-b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-110
+c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-124
|
-|---b: New For Each(false,false)[bag] - scope-109
+|---c: New For Each(false,false)[bag] - scope-123
| |
- | Project[int][0] - scope-103
+ | Project[tuple][0] - scope-117
| |
- | POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-107
+ | POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-121
| |
- | |---Project[bag][1] - scope-144
+ | |---Project[bag][1] - scope-161
|
- |---b: Package(CombinerPackager)[tuple]{int} - scope-100
+ |---c: Package(CombinerPackager)[tuple]{tuple} - scope-113
Modified: pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3.gld
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3.gld?rev=1642132&r1=1642131&r2=1642132&view=diff
==============================================================================
--- pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3.gld (original)
+++ pig/branches/spark/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-3.gld Thu Nov 27 12:49:54 2014
@@ -2,7 +2,7 @@
# There are 1 DAGs in the session
#--------------------------------------------------
#--------------------------------------------------
-# TEZ DAG plan: scope-46
+# TEZ DAG plan: pig-0_scope-0
#--------------------------------------------------
Tez vertex scope-38 -> Tez vertex scope-42,Tez vertex scope-45,
Tez vertex scope-45
@@ -10,39 +10,39 @@ Tez vertex scope-42
Tez vertex scope-38
# Plan on vertex
-a: Split - scope-87
+a: Split - scope-86
| |
-| b: Local Rearrange[tuple]{int}(false) - scope-59 -> scope-42
+| b: Local Rearrange[tuple]{int}(false) - scope-58 -> scope-42
| | |
-| | Project[int][0] - scope-61
+| | Project[int][0] - scope-60
| |
-| |---b: New For Each(false,false)[bag] - scope-47
+| |---b: New For Each(false,false)[bag] - scope-46
| | |
-| | Project[int][0] - scope-48
+| | Project[int][0] - scope-47
| | |
-| | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-49
+| | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-48
| | |
-| | |---Project[bag][0] - scope-50
+| | |---Project[bag][0] - scope-49
| | |
-| | |---Project[bag][1] - scope-51
+| | |---Project[bag][1] - scope-50
| |
-| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-62
+| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-61
| |
-| c: Local Rearrange[tuple]{tuple}(false) - scope-78 -> scope-45
+| c: Local Rearrange[tuple]{tuple}(false) - scope-77 -> scope-45
| | |
-| | Project[tuple][0] - scope-81
+| | Project[tuple][0] - scope-80
| |
-| |---c: New For Each(false,false)[bag] - scope-66
+| |---c: New For Each(false,false)[bag] - scope-65
| | |
-| | Project[tuple][0] - scope-67
+| | Project[tuple][0] - scope-66
| | |
-| | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-68
+| | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-67
| | |
-| | |---Project[bag][1] - scope-69
+| | |---Project[bag][1] - scope-68
| | |
-| | |---Project[bag][1] - scope-70
+| | |---Project[bag][1] - scope-69
| |
-| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-82
+| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-81
|
|---a: New For Each(false,false)[bag] - scope-7
| |
@@ -57,19 +57,19 @@ a: Split - scope-87
|---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
Tez vertex scope-45
# Combine plan on edge <scope-38>
-c: Local Rearrange[tuple]{tuple}(false) - scope-83 -> scope-45
+c: Local Rearrange[tuple]{tuple}(false) - scope-82 -> scope-45
| |
-| Project[tuple][0] - scope-86
+| Project[tuple][0] - scope-85
|
-|---c: New For Each(false,false)[bag] - scope-71
+|---c: New For Each(false,false)[bag] - scope-70
| |
- | Project[tuple][0] - scope-72
+ | Project[tuple][0] - scope-71
| |
- | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-73
+ | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-72
| |
- | |---Project[bag][1] - scope-74
+ | |---Project[bag][1] - scope-73
|
- |---c: Package(CombinerPackager)[tuple]{tuple} - scope-77
+ |---c: Package(CombinerPackager)[tuple]{tuple} - scope-76
# Plan on vertex
c: Store(file:///tmp/output/c:org.apache.pig.builtin.PigStorage) - scope-37
|
@@ -79,24 +79,24 @@ c: Store(file:///tmp/output/c:org.apache
| |
| POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-34
| |
- | |---Project[bag][1] - scope-75
+ | |---Project[bag][1] - scope-74
|
|---c: Package(CombinerPackager)[tuple]{tuple} - scope-26
Tez vertex scope-42
# Combine plan on edge <scope-38>
-b: Local Rearrange[tuple]{int}(false) - scope-63 -> scope-42
+b: Local Rearrange[tuple]{int}(false) - scope-62 -> scope-42
| |
-| Project[int][0] - scope-65
+| Project[int][0] - scope-64
|
-|---b: New For Each(false,false)[bag] - scope-52
+|---b: New For Each(false,false)[bag] - scope-51
| |
- | Project[int][0] - scope-53
+ | Project[int][0] - scope-52
| |
- | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-54
+ | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-53
| |
- | |---Project[bag][1] - scope-55
+ | |---Project[bag][1] - scope-54
|
- |---b: Package(CombinerPackager)[tuple]{int} - scope-58
+ |---b: Package(CombinerPackager)[tuple]{int} - scope-57
# Plan on vertex
b: Store(file:///tmp/output/b:org.apache.pig.builtin.PigStorage) - scope-22
|
@@ -106,6 +106,6 @@ b: Store(file:///tmp/output/b:org.apache
| |
| POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-19
| |
- | |---Project[bag][1] - scope-56
+ | |---Project[bag][1] - scope-55
|
|---b: Package(CombinerPackager)[tuple]{int} - scope-12