You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by zl...@apache.org on 2017/03/30 02:01:01 UTC
svn commit: r1789423 - in /pig/branches/spark/test/e2e/pig/tests:
bigdata.conf cmdline.conf grunt.conf hcat.conf multiquery.conf
negative.conf nightly.conf orc.conf turing_jython.conf
Author: zly
Date: Thu Mar 30 02:01:01 2017
New Revision: 1789423
URL: http://svn.apache.org/viewvc?rev=1789423&view=rev
Log:
PIG-5158:Several e2e tests are marked to run only in Tez or MR mode only(Nandor via Liyun)
Modified:
pig/branches/spark/test/e2e/pig/tests/bigdata.conf
pig/branches/spark/test/e2e/pig/tests/cmdline.conf
pig/branches/spark/test/e2e/pig/tests/grunt.conf
pig/branches/spark/test/e2e/pig/tests/hcat.conf
pig/branches/spark/test/e2e/pig/tests/multiquery.conf
pig/branches/spark/test/e2e/pig/tests/negative.conf
pig/branches/spark/test/e2e/pig/tests/nightly.conf
pig/branches/spark/test/e2e/pig/tests/orc.conf
pig/branches/spark/test/e2e/pig/tests/turing_jython.conf
Modified: pig/branches/spark/test/e2e/pig/tests/bigdata.conf
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/e2e/pig/tests/bigdata.conf?rev=1789423&r1=1789422&r2=1789423&view=diff
==============================================================================
--- pig/branches/spark/test/e2e/pig/tests/bigdata.conf (original)
+++ pig/branches/spark/test/e2e/pig/tests/bigdata.conf Thu Mar 30 02:01:01 2017
@@ -24,7 +24,7 @@
$cfg = {
'driver' => 'Pig',
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'groups' => [
{
Modified: pig/branches/spark/test/e2e/pig/tests/cmdline.conf
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/e2e/pig/tests/cmdline.conf?rev=1789423&r1=1789422&r2=1789423&view=diff
==============================================================================
--- pig/branches/spark/test/e2e/pig/tests/cmdline.conf (original)
+++ pig/branches/spark/test/e2e/pig/tests/cmdline.conf Thu Mar 30 02:01:01 2017
@@ -254,7 +254,7 @@ dump A;\,
{
'name' => 'Warning',
'floatpostprocess' => 0,
- 'execonly' => 'mapred,tez', # Warnings use counters, which don't work in local mode
+ 'execonly' => 'mapred,tez,spark', # Warnings use counters, which don't work in local mode
'delimiter' => ' ',
'tests' => [
Modified: pig/branches/spark/test/e2e/pig/tests/grunt.conf
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/e2e/pig/tests/grunt.conf?rev=1789423&r1=1789422&r2=1789423&view=diff
==============================================================================
--- pig/branches/spark/test/e2e/pig/tests/grunt.conf (original)
+++ pig/branches/spark/test/e2e/pig/tests/grunt.conf Thu Mar 30 02:01:01 2017
@@ -43,13 +43,13 @@ $cfg = {
},{
'num' => 2,
'pig' => "pwd",
- 'execonly' => 'mapred,tez', # don't have a clue what their cwd will be for local mode
+ 'execonly' => 'mapred,tez,spark', # don't have a clue what their cwd will be for local mode
'expected_out_regex' => "/user",
'rc' => 0
},{
'num' => 3,
'pig' => "ls .",
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'expected_out_regex' => "/user",
'rc' => 0
},{
Modified: pig/branches/spark/test/e2e/pig/tests/hcat.conf
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/e2e/pig/tests/hcat.conf?rev=1789423&r1=1789422&r2=1789423&view=diff
==============================================================================
--- pig/branches/spark/test/e2e/pig/tests/hcat.conf (original)
+++ pig/branches/spark/test/e2e/pig/tests/hcat.conf Thu Mar 30 02:01:01 2017
@@ -24,7 +24,7 @@
$cfg = {
'driver' => 'Pig',
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'groups' => [
{
Modified: pig/branches/spark/test/e2e/pig/tests/multiquery.conf
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/e2e/pig/tests/multiquery.conf?rev=1789423&r1=1789422&r2=1789423&view=diff
==============================================================================
--- pig/branches/spark/test/e2e/pig/tests/multiquery.conf (original)
+++ pig/branches/spark/test/e2e/pig/tests/multiquery.conf Thu Mar 30 02:01:01 2017
@@ -529,7 +529,7 @@ $cfg = {
# Streaming in demux
{
'num' => 2,
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'pig' => q#
define CMD `perl GroupBy.pl '\t' 0` ship(':SCRIPTHOMEPATH:/GroupBy.pl');
A = load ':INPATH:/singlefile/studenttab10k';
@@ -547,7 +547,7 @@ $cfg = {
# Streaming in nested demux
{
'num' => 3,
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'pig' => q#
define CMD `perl GroupBy.pl '\t' 0` ship(':SCRIPTHOMEPATH:/GroupBy.pl');
A = load ':INPATH:/singlefile/studenttab10k';
Modified: pig/branches/spark/test/e2e/pig/tests/negative.conf
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/e2e/pig/tests/negative.conf?rev=1789423&r1=1789422&r2=1789423&view=diff
==============================================================================
--- pig/branches/spark/test/e2e/pig/tests/negative.conf (original)
+++ pig/branches/spark/test/e2e/pig/tests/negative.conf Thu Mar 30 02:01:01 2017
@@ -312,7 +312,7 @@ dump B;#,
{
# Define uses using non-existent command (autoship)
'num' => 1,
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'java_params' => ['-Dopt.fetch=false'],
'pig' => q\
define CMD `perl PigStreamingNotThere.pl`;
@@ -411,7 +411,7 @@ dump B;\,
# Streaming application fails in the beginning of processing
# NEED TO CHECK STDERR MANUALLY FOR NOW
'num' => 1,
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'pig' => q\
define CMD `perl PigStreamingBad.pl start` ship(':SCRIPTHOMEPATH:/PigStreamingBad.pl') stderr('CMD' limit 1);
A = load ':INPATH:/singlefile/studenttab10k';
@@ -423,7 +423,7 @@ store B into ':OUTPATH:';\,
# Streaming application fails in the middle of processing
# NEED TO CHECK STDERR MANUALLY FOR NOW
'num' => 2,
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'pig' => q\
define CMD `perl PigStreamingBad.pl middle` ship(':SCRIPTHOMEPATH:/PigStreamingBad.pl') stderr('CMD' limit 1);
A = load ':INPATH:/singlefile/studenttab10k';
@@ -436,7 +436,7 @@ store B into ':OUTPATH:';\,
# bring logs to dfs
# NEED TO CHECK STDERR MANUALLY FOR NOW
'num' => 3,
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'pig' => q\
define CMD `perl PigStreamingBad.pl end` ship(':SCRIPTHOMEPATH:/PigStreamingBad.pl') stderr('CMD' limit 1);
A = load ':INPATH:/singlefile/studenttab10k';
@@ -449,7 +449,7 @@ store B into ':OUTPATH:';\,
# bring logs to dfs
# NEED TO CHECK STDERR MANUALLY FOR NOW
'num' => 4,
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'pig' => q\
define CMD `perl DieRandomly.pl 10000 2` ship(':SCRIPTHOMEPATH:/DieRandomly.pl') stderr('CMD' limit 1);
A = load ':INPATH:/singlefile/studenttab10k';
@@ -466,7 +466,7 @@ store B into ':OUTPATH:';\,
{
# Invalid deserializer - throws exception
'num' => 1,
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'pig' => q\
register :FUNCPATH:/testudf.jar;
define CMD `perl PigStreaming.pl` input(stdin) output(stdout using org.apache.pig.test.udf.streaming.DumpStreamerBad) ship(':SCRIPTHOMEPATH:/PigStreaming.pl');
@@ -478,7 +478,7 @@ store B into ':OUTPATH:';\,
{
# Invalid serializer - throws exception
'num' => 2,
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'pig' => q\
define CMD `perl PigStreamingDepend.pl` input(stdin using StringStoreBad) ship(':SCRIPTHOMEPATH:/PigStreamingDepend.pl', ':SCRIPTHOMEPATH:/PigStreamingModule.pm');
A = load ':INPATH:/singlefile/studenttab10k';
Modified: pig/branches/spark/test/e2e/pig/tests/nightly.conf
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/e2e/pig/tests/nightly.conf?rev=1789423&r1=1789422&r2=1789423&view=diff
==============================================================================
--- pig/branches/spark/test/e2e/pig/tests/nightly.conf (original)
+++ pig/branches/spark/test/e2e/pig/tests/nightly.conf Thu Mar 30 02:01:01 2017
@@ -2289,7 +2289,7 @@ store b into ':OUTPATH:';\,
},
{
'num' => 12,
- 'execonly' => 'tez', #Limit_5 was not able to test on tez.
+ 'execonly' => 'tez,spark', #Limit_5 was not able to test on tez.
'pig' =>q\a = load ':INPATH:/singlefile/studenttab10k';
b = load ':INPATH:/singlefile/studenttab10k';
a1 = foreach a generate $0, $1;
@@ -2954,7 +2954,7 @@ store d into ':OUTPATH:';\,
# Merge-join with one file across multiple blocks
{
'num' => 8,
- 'execonly' => 'mapred,tez', # since this join will run out of memory in local mode
+ 'execonly' => 'mapred,tez,spark', # since this join will run out of memory in local mode
'floatpostprocess' => 1,
'delimiter' => ' ',
'pig' => q\a = load ':INPATH:/singlefile/votertab10k';
@@ -3622,7 +3622,7 @@ store b into ':OUTPATH:';\,
'tests' => [
{
'num' => 1,
- 'execonly' => 'mapred,tez', # studenttab20m not available in local mode
+ 'execonly' => 'mapred,tez,spark', # studenttab20m not available in local mode
'pig' => q\
a = load ':INPATH:/singlefile/studenttab20m' using PigStorage() as (name, age, gpa);
b = foreach a generate age;
@@ -4367,7 +4367,7 @@ store b into ':OUTPATH:';\,
{
# test group
'num' => 1,
- 'execonly' => 'mapred,tez', # since this join will run out of memory in local mode
+ 'execonly' => 'mapred,tez,spark', # since this join will run out of memory in local mode
'pig' => q\register :FUNCPATH:/testudf.jar;
a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age:int, gpa);
b = group a by age PARTITION BY org.apache.pig.test.utils.SimpleCustomPartitioner2 parallel 2;
@@ -4958,7 +4958,7 @@ store C into ':OUTPATH:';\,
{
'num' => 1,
'java_params' => ['-Dopt.fetch=false'],
- 'execonly' => 'mapred,tez', # since distributed cache is not supported in local mode
+ 'execonly' => 'mapred,tez,spark', # since distributed cache is not supported in local mode
'pig' => q?
register :FUNCPATH:/testudf.jar;
define udfdc org.apache.pig.test.udf.evalfunc.Udfcachetest(':INPATH:/singlefile/votertab10k#foodle');
@@ -5200,7 +5200,7 @@ store C into ':OUTPATH:';\,
}, {
# PIG-2576
'num' => 4,
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'pig' => q?register :FUNCPATH:/testudf.jar;
define printconf org.apache.pig.test.udf.evalfunc.UdfContextFrontend('dummy');
a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
@@ -5255,7 +5255,7 @@ store C into ':OUTPATH:';\,
],
},{
'name' => 'Bloom',
- 'execonly' => 'mapred,tez', # distributed cache does not work in local mode
+ 'execonly' => 'mapred,tez', # distributed cache does not work in local mode, bloom is not implemented for Spark(PIG-5117)
'tests' => [
{
'num' => 1,
@@ -5700,7 +5700,7 @@ store a into ':OUTPATH:';\,
'tests' => [
{
'num' => 1,
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'pig' => q\
SET default_parallel 7;
SET mapreduce.input.fileinputformat.split.maxsize '300';
@@ -5717,7 +5717,7 @@ store a into ':OUTPATH:';\,
\,
}, {
'num' =>2,
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'pig' => q\
SET default_parallel 9;
SET mapreduce.input.fileinputformat.split.maxsize '300';
@@ -5734,7 +5734,7 @@ store a into ':OUTPATH:';\,
\,
}, {
'num' =>3,
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'pig' => q\
SET default_parallel 7;
SET mapreduce.input.fileinputformat.split.maxsize '300';
@@ -5751,7 +5751,7 @@ store a into ':OUTPATH:';\,
\,
}, {
'num' => 4,
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'pig' => q\
SET default_parallel 5;
SET mapreduce.input.fileinputformat.split.maxsize '300';
@@ -5769,7 +5769,7 @@ store a into ':OUTPATH:';\,
\,
}, {
'num' => 5,
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'pig' => q\
SET default_parallel 5;
SET mapreduce.input.fileinputformat.split.maxsize '300';
@@ -5792,7 +5792,7 @@ store a into ':OUTPATH:';\,
\,
}, {
'num' => 6,
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'pig' => q\
SET mapreduce.input.fileinputformat.split.maxsize '300';
SET pig.splitCombination false;
@@ -5816,7 +5816,7 @@ store a into ':OUTPATH:';\,
'tests' => [
{
'num' => 1,
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'pig' => q\
SET mapreduce.input.fileinputformat.split.maxsize '300';
SET pig.splitCombination false;
@@ -5833,7 +5833,7 @@ store a into ':OUTPATH:';\,
\,
}, {
'num' => 2,
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'pig' => q\
SET mapreduce.input.fileinputformat.split.maxsize '300';
SET pig.splitCombination false;
@@ -5850,7 +5850,7 @@ store a into ':OUTPATH:';\,
\,
}, {
'num' => 3,
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'pig' => q\
SET mapreduce.input.fileinputformat.split.maxsize '300';
SET pig.splitCombination false;
@@ -5867,7 +5867,7 @@ store a into ':OUTPATH:';\,
\,
}, {
'num' => 4,
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'pig' => q\
SET mapreduce.input.fileinputformat.split.maxsize '300';
SET pig.splitCombination false;
@@ -5887,7 +5887,7 @@ store a into ':OUTPATH:';\,
\,
}, {
'num' => 5,
- 'execonly' => 'mapred,tez',
+ 'execonly' => 'mapred,tez,spark',
'pig' => q\
SET default_parallel 9;
SET mapreduce.input.fileinputformat.split.maxsize '300';
Modified: pig/branches/spark/test/e2e/pig/tests/orc.conf
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/e2e/pig/tests/orc.conf?rev=1789423&r1=1789422&r2=1789423&view=diff
==============================================================================
--- pig/branches/spark/test/e2e/pig/tests/orc.conf (original)
+++ pig/branches/spark/test/e2e/pig/tests/orc.conf Thu Mar 30 02:01:01 2017
@@ -139,7 +139,7 @@ store b into ':OUTPATH:';\,
{
'num' => 2,
'notmq' => 1,
- 'execonly' => 'mapred,tez', # studenttab20m not available in local mode
+ 'execonly' => 'mapred,tez,spark', # studenttab20m not available in local mode
'pig' => q\
a = load ':INPATH:/singlefile/studenttab20m' as (name:chararray, age:int, gpa:float);
b = order a by age desc parallel 4;
Modified: pig/branches/spark/test/e2e/pig/tests/turing_jython.conf
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/e2e/pig/tests/turing_jython.conf?rev=1789423&r1=1789422&r2=1789423&view=diff
==============================================================================
--- pig/branches/spark/test/e2e/pig/tests/turing_jython.conf (original)
+++ pig/branches/spark/test/e2e/pig/tests/turing_jython.conf Thu Mar 30 02:01:01 2017
@@ -452,9 +452,9 @@ result = P.bind({'in1':input1, 'in2':inp
,'rc'=> 0
}, {
- # illustrate() on a complex query
+ # illustrate() on a complex query
'num' => 2
- ,'execonly' => 'mapred,local' #TODO: PIG-3993: Illustrate is yet to be implemented in Tez
+ ,'execonly' => 'mapred,local' #TODO: PIG-3993,PIG-5204: Illustrate is yet to be implemented in Tez and in Spark
,'pig' => q\#!/usr/bin/python
from org.apache.pig.scripting import Pig