You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by kn...@apache.org on 2016/11/21 17:30:28 UTC
svn commit: r1770724 - in /pig/trunk: CHANGES.txt
test/e2e/pig/tests/nightly.conf
Author: knoguchi
Date: Mon Nov 21 17:30:28 2016
New Revision: 1770724
URL: http://svn.apache.org/viewvc?rev=1770724&view=rev
Log:
PIG-5062: Allow Native e2e tests to run in parallel (knoguchi)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/test/e2e/pig/tests/nightly.conf
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1770724&r1=1770723&r2=1770724&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Nov 21 17:30:28 2016
@@ -59,6 +59,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-5062: Allow Native e2e tests to run in parallel (knoguchi)
+
PIG-5060: TestPigRunner.testDisablePigCounters2 failing with tez (knoguchi)
PIG-5056: Fix AvroStorage writing enums (szita via daijy)
Modified: pig/trunk/test/e2e/pig/tests/nightly.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/nightly.conf?rev=1770724&r1=1770723&r2=1770724&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/nightly.conf (original)
+++ pig/trunk/test/e2e/pig/tests/nightly.conf Mon Nov 21 17:30:28 2016
@@ -4243,40 +4243,32 @@ store e into ':OUTPATH:';\,
# test common
'num' => 1,
'pig' => q\
-rmf table_testNativeMRJobSimple_input
-rmf table_testNativeMRJobSimple_output
a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
-b = native ':MAPREDJARS:/hadoop-examples.jar' Store a into 'table_testNativeMRJobSimple_input' Load 'table_testNativeMRJobSimple_output' `wordcount table_testNativeMRJobSimple_input table_testNativeMRJobSimple_output`;
+b = native ':MAPREDJARS:/hadoop-examples.jar' Store a into ':OUTPATH:.intermediate.1' Load ':OUTPATH:.intermediate.2' `wordcount :OUTPATH:.intermediate.1 :OUTPATH:.intermediate.2`;
store b into ':OUTPATH:';\,
'notmq' => 1,
'verify_pig_script' => q\
-rmf table_testNativeMRJobSimple_input
-rmf table_testNativeMRJobSimple_output
a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
-b = mapreduce ':MAPREDJARS:/hadoop-examples.jar' Store a into 'table_testNativeMRJobSimple_input' Load 'table_testNativeMRJobSimple_output' `wordcount table_testNativeMRJobSimple_input table_testNativeMRJobSimple_output`;
+b = mapreduce ':MAPREDJARS:/hadoop-examples.jar' Store a into ':OUTPATH:.intermediate.1' Load ':OUTPATH:.intermediate.2' `wordcount :OUTPATH:.intermediate.1 :OUTPATH:.intermediate.2`;
store b into ':OUTPATH:';\,
},
{
# test complex
'num' => 2,
'pig' => q\
-rmf table_testNativeMRJobSimple_input
-rmf table_testNativeMRJobSimple_output
a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
b = foreach a generate name;
c = distinct b;
-d = native ':MAPREDJARS:/hadoop-examples.jar' Store c into 'table_testNativeMRJobSimple_input' Load 'table_testNativeMRJobSimple_output' as (name:chararray, count: int) `wordcount table_testNativeMRJobSimple_input table_testNativeMRJobSimple_output`;
+d = native ':MAPREDJARS:/hadoop-examples.jar' Store c into ':OUTPATH:.intermediate.1' Load ':OUTPATH:.intermediate.2' as (name:chararray, count: int) `wordcount :OUTPATH:.intermediate.1 :OUTPATH:.intermediate.2`;
e = order d by name;
store e into ':OUTPATH:';\,
'sortArgs' => ['-t', ' '],
'notmq' => 1,
'verify_pig_script' => q\
-rmf table_testNativeMRJobSimple_input
-rmf table_testNativeMRJobSimple_output
a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
b = foreach a generate name;
c = distinct b;
-d = mapreduce ':MAPREDJARS:/hadoop-examples.jar' Store c into 'table_testNativeMRJobSimple_input' Load 'table_testNativeMRJobSimple_output' as (name:chararray, count: int) `wordcount table_testNativeMRJobSimple_input table_testNativeMRJobSimple_output`;
+d = mapreduce ':MAPREDJARS:/hadoop-examples.jar' Store c into ':OUTPATH:.intermediate.1' Load ':OUTPATH:.intermediate.2' as (name:chararray, count: int) `wordcount :OUTPATH:.intermediate.1 :OUTPATH:.intermediate.2`;
e = order d by name;
store e into ':OUTPATH:';\,
},
@@ -4284,16 +4276,12 @@ store e into ':OUTPATH:';\,
# test streaming
'num' => 3,
'pig' => q\
-rmf table_testNativeMRJobSimple_input
-rmf table_testNativeMRJobSimple_output
a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
-b = mapreduce ':MAPREDJARS:/hadoop-streaming.jar' Store a into 'table_testNativeMRJobSimple_input' Load 'table_testNativeMRJobSimple_output' as (name:chararray, count: int) `-input table_testNativeMRJobSimple_input -output table_testNativeMRJobSimple_output -mapper cat -reducer wc`;
+b = mapreduce ':MAPREDJARS:/hadoop-streaming.jar' Store a into ':OUTPATH:.intermediate.1' Load ':OUTPATH:.intermediate.2' as (name:chararray, count: int) `-input :OUTPATH:.intermediate.1 -output :OUTPATH:.intermediate.2 -mapper cat -reducer wc`;
store b into ':OUTPATH:';\,
'pig23' => q\
-rmf table_testNativeMRJobSimple_input
-rmf table_testNativeMRJobSimple_output
a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
-b = mapreduce ':MAPREDJARS:/hadoop-0.23.0-streaming.jar' Store a into 'table_testNativeMRJobSimple_input' Load 'table_testNativeMRJobSimple_output' as (name:chararray, count: int) `-input table_testNativeMRJobSimple_input -output table_testNativeMRJobSimple_output -mapper cat -reducer wc`;
+b = mapreduce ':MAPREDJARS:/hadoop-0.23.0-streaming.jar' Store a into ':OUTPATH:.intermediate.1' Load ':OUTPATH:.intermediate.2' as (name:chararray, count: int) `-input :OUTPATH:.intermediate.1 -output :OUTPATH:.intermediate.2 -mapper cat -reducer wc`;
store b into ':OUTPATH:';\,
'notmq' => 1,
},