You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2011/06/15 23:54:51 UTC
svn commit: r1136214 - in /pig/trunk: ./ test/e2e/pig/ test/e2e/pig/conf/
test/e2e/pig/drivers/ test/e2e/pig/tests/
Author: gates
Date: Wed Jun 15 21:54:50 2011
New Revision: 1136214
URL: http://svn.apache.org/viewvc?rev=1136214&view=rev
Log:
PIG-1950 e2e test harness needs to be able to compare to previous version of Pig
Modified:
pig/trunk/CHANGES.txt
pig/trunk/test/e2e/pig/Makefile
pig/trunk/test/e2e/pig/conf/default.conf
pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm
pig/trunk/test/e2e/pig/tests/cmdline.conf
pig/trunk/test/e2e/pig/tests/multiquery.conf
pig/trunk/test/e2e/pig/tests/negative.conf
pig/trunk/test/e2e/pig/tests/nightly.conf
pig/trunk/test/e2e/pig/tests/turing_jython.conf
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1136214&r1=1136213&r2=1136214&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Jun 15 21:54:50 2011
@@ -24,6 +24,9 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-1950: e2e test harness needs to be able to compare to previous version of
+Pig (gates)
+
PIG-536: the shell script 'pig' does not work if PIG_HOME has the word 'hadoop' in it's directory (miguno via olgan)
PIG-2108 e2e test harness needs to be able to mark certain tests as ignored
Modified: pig/trunk/test/e2e/pig/Makefile
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/Makefile?rev=1136214&r1=1136213&r2=1136214&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/Makefile (original)
+++ pig/trunk/test/e2e/pig/Makefile Wed Jun 15 21:54:50 2011
@@ -24,12 +24,13 @@ DEPLOYERS = $(DEPLOYER_DIR)/ExistingClus
TEST_DIR = ./tests
-TESTS = $(TEST_DIR)/bigdata.conf $(TEST_DIR)/cmdline.conf \
- $(TEST_DIR)/grunt.conf \
+TESTS = $(TEST_DIR)/cmdline.conf \
$(TEST_DIR)/multiquery.conf $(TEST_DIR)/negative.conf \
$(TEST_DIR)/nightly.conf $(TEST_DIR)/streaming.conf \
$(TEST_DIR)/turing_jython.conf
+# TODO Add in bigdata and macro tests. bigdata should be in separate target
+
CONF_DIR = ./conf
CONF_FILES = $(CONF_DIR)/default.conf $(CONF_DIR)/existing_deployer.conf
@@ -45,50 +46,15 @@ TOOLS = $(TEST_TOOL_DIR)/explainchk.pl
ARCHIVE = pigtests.tar
-TEST_DIST_DIR = testdist
-
-TESTS_TO_RUN ?= -t Checkin -t LoaderDefaultDir -t LoaderPigStorageArg \
- -t FilterBoolean -t FilterEq -t FilterMatches -t FilterUdf \
- -t CoGroupFlatten -t Join -t Distinct -t Cross -t Union \
- -t Glob -t Arithmetic -t Regression -t Unicode -t Parameters \
- -t ImplicitSplit -t describe -t MissingColumns -t Aliases \
- -t Lineage -t Casts -t ClassResolution \
- -t MergeJoin -t SkewedJoin -t CollectedGroup -t Accumulator \
- -t PruneColumns -t Bzip -t Scalar -t Scripting -t NestedDescribe \
- -t UdfDistributedCache -t Describe_cmdline -t Unicode_cmdline \
- -t Warning -t NoSuchFile -t BadFunc -t FileExists \
- -t NegForeach -t NotAllowed -t UdfException \
- -t LineageErrors -t ComputeSpec -t JobManagement \
- -t StreamingIO -t NonStreaming -t StreamingPerformance \
- -t RaceConditions -t StreamingLocal -t Jython_Checkin
+TEST_DIST_DIR ?= testdist
+# TODO
+# Make grunt tests works
+# Make Jython tests work, since we don't have an old version for some of them
+# to work aginst
+# Make Macro tests work, can hand expand and test
-
# Not yet working:
-# LoaderBinStorage - results doctored
-# LoaderTestLoader - results doctored
-# GroupAggFunc - results doctored
-# EvalFunc - results doctored
-# CoGroup - results doctored
-# Foreach - results doctored
-# Order - results doctored
-# Bincond - results doctored
-# Types - results doctored
-# Limit - results doctored
-# Split - results doctored
-# SecondarySort - results doctored
-# Native - results doctored
-# Partitioner - issues unclear
-# CastScalar - issues unclear
-# udf_TOBAGandTOTUPLE -results doctored
-# MergeOperator - issues unclear
-# Grunt - needs reworked with different inputs
-# MultiQuery - results doctored
-# SyntaxErrors - needs streaming resources
-# CmdErrors - needs streaming resources
-# StreamingErrors - needs streaming resources
-# IOErrors - needs streaming resources
-# StreamingLocalErrors - needs streaming resources
# Jython_* - needs switch from java to bin/pig
UDF_DIR = udfs
@@ -117,6 +83,10 @@ build_udfs:
(cd $(JAVA_UDF_DIR); ant -Dpig.jarfile=$${PH_PIG}/pig.jar)
pretest: $(ARCHIVE) build_udfs
+ @(if [ "$${PH_OLDPIG}x" == "x" ] ; then echo \
+ "You must set the environment variable PH_OLDPIG" \
+ "to the directory your old version of Pig is installed in" \
+ "before running the tests"; 1; fi)
@(if [ "$${PH_CLUSTER}x" == "x" ] ; then echo \
"You must set the environment variable PH_CLUSTER" \
"to the directory that contains your hadoop-site.xml" \
Modified: pig/trunk/test/e2e/pig/conf/default.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/conf/default.conf?rev=1136214&r1=1136213&r2=1136214&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/conf/default.conf (original)
+++ pig/trunk/test/e2e/pig/conf/default.conf Wed Jun 15 21:54:50 2011
@@ -47,6 +47,7 @@ $cfg = {
, 'funcjarPath' => "$ENV{PH_ROOT}/lib/java"
, 'paramPath' => "$ENV{PH_ROOT}/paramfiles"
, 'pigpath' => "$ENV{PH_PIG}"
+ , 'oldpigpath' => "$ENV{PH_OLDPIG}"
#HADOOP
, 'hadoopHome' => "$ENV{PH_ROOT}/lib"
Modified: pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm?rev=1136214&r1=1136213&r2=1136214&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm (original)
+++ pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm Wed Jun 15 21:54:50 2011
@@ -175,7 +175,7 @@ sub globalSetup
}
- print $log join(" ", @cmd);
+ # print $log join(" ", @cmd);
if($self->{'exectype'} eq "mapred")
{
@@ -365,6 +365,8 @@ sub getPigCmd($$$)
}
+ print $log "Returning Pig command " . join(" ", @pigCmd) . "\n";
+ print $log "With PIG_CLASSPATH set to " . $ENV{'PIG_CLASSPATH'} . "\n";
return @pigCmd;
}
@@ -545,23 +547,44 @@ sub generateBenchmark
my %result;
- $self->parseSqlQueries($testCmd);
- my @SQLQuery = @{$testCmd->{'queries'}};
+ if (defined $testCmd->{'verify_with_pig'}) {
+ # Verify against an old version of Pig instead of against the database
+ my %modifiedTestCmd = %{$testCmd};
+ # They can either test against a previous version of pig or against the
+ # current version
+ if ($testCmd->{'verify_pig_version'} == "old") {
+ # Change so we're looking at the old version of Pig
+ $modifiedTestCmd{'pigpath'} = $testCmd->{'oldpigpath'};
+ }
+ if (defined $testCmd->{'verify_pig_script'}) {
+ $modifiedTestCmd{'pig'} = $testCmd->{'verify_pig_script'};
+ }
+ # Modify the test number so we don't run over the actual test output
+ # and logs
+ $modifiedTestCmd{'num'} = $testCmd->{'num'} . "_benchmark";
+ return $self->runPig(\%modifiedTestCmd, $log, 1);
+ } else {
+ # Verify against the database
+ $self->parseSqlQueries($testCmd);
+ my @SQLQuery = @{$testCmd->{'queries'}};
- if ($#SQLQuery == 0) {
- my $outfile = $self->generateSingleSQLBenchmark($testCmd, $SQLQuery[0], undef, $log);
- $result{'output'} = $outfile;
- } else {
- my @outfiles = ();
- for (my $id = 0; $id < ($#SQLQuery + 1); $id++) {
- my $sql = $SQLQuery[$id];
- my $outfile = $self->generateSingleSQLBenchmark($testCmd, $sql, ($id+1), $log);
- push(@outfiles, $outfile);
- }
- $result{'outputs'} = \@outfiles;
- }
+ if ($#SQLQuery == 0) {
+ my $outfile = $self->generateSingleSQLBenchmark($testCmd,
+ $SQLQuery[0], undef, $log);
+ $result{'output'} = $outfile;
+ } else {
+ my @outfiles = ();
+ for (my $id = 0; $id < ($#SQLQuery + 1); $id++) {
+ my $sql = $SQLQuery[$id];
+ my $outfile = $self->generateSingleSQLBenchmark($testCmd,
+ $sql, ($id+1), $log);
+ push(@outfiles, $outfile);
+ }
+ $result{'outputs'} = \@outfiles;
+ }
- return \%result;
+ return \%result;
+ }
}
sub generateSingleSQLBenchmark
Modified: pig/trunk/test/e2e/pig/tests/cmdline.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/cmdline.conf?rev=1136214&r1=1136213&r2=1136214&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/cmdline.conf (original)
+++ pig/trunk/test/e2e/pig/tests/cmdline.conf Wed Jun 15 21:54:50 2011
@@ -84,7 +84,7 @@ describe A;\,
'pig' => q\
A = load ':INPATH:/singlefile/studenttabcomplex10k' using PigStorage() as (m:map[],x,y);
describe A;\,
- 'expected_out_regex' =>"A: {m: map\\[\\],x: bytearray,y: bytearray}",
+ 'expected_out_regex' =>"A: {m: map\\[ \\],x: bytearray,y: bytearray}",
},
#JIRA[PIG-373]
@@ -99,7 +99,7 @@ describe B;\,
#Expect
# A : { m:map, x:bytearray, y:bytearray }
# B : { m :map }
- 'expected_out_regex' => "A: {m: map\\[\\],x: bytearray,y: bytearray}\nB: {m: map\\[\\]}",
+ 'expected_out_regex' => "A: {m: map\\[ \\],x: bytearray,y: bytearray}\nB: {m: map\\[ \\]}",
},
#JIRA[PIG-373]
#JIRA[PIG-405]
@@ -113,7 +113,7 @@ describe B;\,
#Expect
# A : { m:map, x:bytearray, y:bytearray }
# B : { m :map }
- 'expected_out_regex' => "A: {m: map\\[\\],x: bytearray,y: bytearray}\nB: {m: map\\[\\]}",
+ 'expected_out_regex' => "A: {m: map\\[ \\],x: bytearray,y: bytearray}\nB: {m: map\\[ \\]}",
},
#JIRA[PIG-373]
#JIRA[PIG-405]
@@ -127,7 +127,7 @@ describe A;\,
#Expect
# A : { m:map, x:bytearray, y:bytearray }
# B : { m :map }
- 'expected_out_regex' => "B: {m: map\\[\\]}\nA: {m: map\\[\\],x: bytearray,y: bytearray}",
+ 'expected_out_regex' => "B: {m: map\\[ \\]}\nA: {m: map\\[ \\],x: bytearray,y: bytearray}",
},
#JIRA[PIG-379]
Modified: pig/trunk/test/e2e/pig/tests/multiquery.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/multiquery.conf?rev=1136214&r1=1136213&r2=1136214&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/multiquery.conf (original)
+++ pig/trunk/test/e2e/pig/tests/multiquery.conf Wed Jun 15 21:54:50 2011
@@ -202,6 +202,8 @@ $cfg = {
select name, count(A.name), count(B.name)
from (select distinct name from studenttab10k where name < 'm') as A
join (select name from studenttab10k where age > 50) as B using (name) group by name;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
# Implicit split with multiple side files.
Modified: pig/trunk/test/e2e/pig/tests/negative.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/negative.conf?rev=1136214&r1=1136213&r2=1136214&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/negative.conf (original)
+++ pig/trunk/test/e2e/pig/tests/negative.conf Wed Jun 15 21:54:50 2011
@@ -289,7 +289,7 @@ define CMD `perl PigStreaming.pl foo -`
A = load ':INPATH:/singlefile/studenttab10k';
B = stream A through CMD;
dump B;#,
- 'expected_err_regex' => "Encountered .*/.* at line 2, column 46.",
+ 'expected_err_regex' => 'Encountered " "\." "\. "" at line 2, column 46.',
},
]
},
@@ -473,11 +473,12 @@ store D into ':OUTPATH:';\,
},
{
'name' => 'StreamingLocalErrors',
+ 'ignore' => 'Until we have exectype working',
'tests' => [
{
# Define uses using non-existent command
'num' => 1,
- 'exectype' => 'local',
+ 'exectype' => 'local',
'pig' => q\
define CMD `perl PigStreamingNotThere.pl`;
A = load ':INPATH:/singlefile/studenttab10k';
Modified: pig/trunk/test/e2e/pig/tests/nightly.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/nightly.conf?rev=1136214&r1=1136213&r2=1136214&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/nightly.conf (original)
+++ pig/trunk/test/e2e/pig/tests/nightly.conf Wed Jun 15 21:54:50 2011
@@ -121,6 +121,8 @@ store b into ':OUTPATH:.intermediate' us
c = load ':OUTPATH:.intermediate' using BinStorage();
store c into ':OUTPATH:' using org.apache.pig.test.udf.storefunc.StringStore();\,
'sql' => "select name, age, gpa from studenttab10k;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
]
},
@@ -136,6 +138,8 @@ a = load ':INPATH:/singlefile/textdoc' u
b = foreach a generate TOKENIZE((chararray)$0);
store b into ':OUTPATH:' using org.apache.pig.test.udf.storefunc.StringStore();\,
'sql' => "select name from textdoc;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
]
},
@@ -573,15 +577,6 @@ c = foreach b generate group.name, group
store c into ':OUTPATH:';\,
'sql' => "select name, age, count(gpa) from studenttab10k group by name, age;",
},
-# Commented out for 1378834 (COUNT(*) syntax may be wrong, but we should detect that and give an error)
-# {
-# 'num' => 4,
-# 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
-#b = group a by name;
-#c = foreach b generate group, COUNT(*);
-#store c into ':OUTPATH:';\,
-# 'sql' => "select name, count(*) from studenttab10k group by name;",
-# },
{
'num' => 5,
'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
@@ -591,9 +586,6 @@ store c into ':OUTPATH:';\,
'sql' => "select count(*) from studenttab10k;",
},
{
- # Results doctored, if you change this query you need to copy the
- # expected results into test/nightly/benchmarks
- # Only until we do sum of ints in ints instead of floats
'num' => 6,
'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
b = group a by name;
@@ -604,6 +596,8 @@ from studenttab10k
group by name;",
'floatpostprocess' => 1,
'delimiter' => ' ',
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
'num' => 7,
@@ -630,8 +624,6 @@ group by name;",
'delimiter' => ' ',
},
{
- # Results doctored, if you change this query you need to copy the
- # expected results into test/nightly/benchmarks
'num' => 9,
'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
b = group a by name;
@@ -642,6 +634,8 @@ from studenttab10k
group by name;",
'floatpostprocess' => 1,
'delimiter' => ' ',
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
'num' => 10,
@@ -707,8 +701,6 @@ store c into ':OUTPATH:';\,
'sql' => "select 3 from studenttab10k where name < 'b';",
},
{
- # Results doctored, if you change this query you need to copy the
- # expected results into test/nightly/benchmarks
'num' => 2,
'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age, gpa);
b = filter a by name lt 'b';
@@ -716,10 +708,10 @@ c = foreach b generate TOKENIZE(name);
d = foreach c generate flatten($0);
store d into ':OUTPATH:';\,
'sql' => "select name from studenttab10k;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
- # Results doctored, if you change this query you need to copy the
- # expected results into test/nightly/benchmarks
'num' => 3,
'pig' => q\register :FUNCPATH:/testudf.jar;
a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
@@ -727,10 +719,10 @@ b = filter a by name lt 'b';
c = foreach b generate org.apache.pig.test.udf.evalfunc.Swap(name, age);
store c into ':OUTPATH:' using org.apache.pig.test.udf.storefunc.StringStore();\,
'sql' => "select age, name from studenttab10k where name < 'b';",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
- # Results doctored, if you change this query you need to copy the
- # expected results into test/nightly/benchmarks
'num' => 4,
'pig' => q\register :FUNCPATH:/testudf.jar;
a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
@@ -738,6 +730,8 @@ b = filter a by name lt 'b';
c = foreach b generate org.apache.pig.test.udf.evalfunc.CreateMap((chararray)name, age);
store c into ':OUTPATH:' using org.apache.pig.test.udf.storefunc.StringStore();\,
'sql' => "select name, age from studenttab10k where name < 'b';",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
}
]
},
@@ -853,6 +847,8 @@ store d into ':OUTPATH:';\,
'sql' => "select a.name, b.name, count(*)
from studenttab10k as a full outer join votertab10k as b using(name)
group by a.name, b.name;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
]
},
@@ -1046,7 +1042,9 @@ d = foreach c {
generate group, flatten(co);
}
store d into ':OUTPATH:';\,
- 'sql' => "select distinct age, gpa from studenttab10k where age < 20 and gpa < 3.0 order by age, gpa;"
+ 'sql' => "select distinct age, gpa from studenttab10k where age < 20 and gpa < 3.0 order by age, gpa;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
# test flatten for map and scalar
@@ -1056,6 +1054,8 @@ a = load ':INPATH:/singlefile/studenttab
b = foreach a generate flatten(name) as n, flatten(org.apache.pig.test.udf.evalfunc.CreateMap((chararray)name, gpa)) as m;
store b into ':OUTPATH:' using org.apache.pig.test.udf.storefunc.StringStore();\,
'sql' => "select name, age, gpa from studenttab10k;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
# test flatten for UDF that returns bag with multiple tuples with multiple columns
@@ -1067,6 +1067,8 @@ store b into ':OUTPATH:';\,
'sql' => "select name, age, gpa from studenttab10k union all select name, gpa, age from studenttab10k;",
'floatpostprocess' => 1,
'delimiter' => ' ',
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
'num' => 8,
@@ -1090,7 +1092,9 @@ d = foreach c {
generate group, flatten(co);
}
store d into ':OUTPATH:';\,
- 'sql' => "select distinct age, gpa from studenttab10k where age < 20 and gpa >= 3.0 and gpa <= 3.5 order by age, gpa;"
+ 'sql' => "select distinct age, gpa from studenttab10k where age < 20 and gpa >= 3.0 and gpa <= 3.5 order by age, gpa;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
# test filter, projection, sort , duplicate elimination
@@ -1261,6 +1265,8 @@ c = foreach b {c1 = order $1 by *; gener
store c into ':OUTPATH:';\,
'sql' => "select name, age, gpa from studenttab10k order by name, age, gpa;",
# 'sortArgs' => ['-t', ' '],
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
'num' => 15,
@@ -1271,6 +1277,8 @@ c = foreach b {c1 = order $1 by * using
store c into ':OUTPATH:';\,
'sql' => "select name, age, gpa from studenttab10k order by name, age desc, gpa desc;",
#'sortArgs' => ['-t', ' ', '-r'],
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
'num' => 16,
@@ -1281,6 +1289,8 @@ c = foreach b {c1 = order $1 by $1 using
store c into ':OUTPATH:';\,
'sql' => "select name, age, gpa from studenttab10k order by name, age desc;",
#'sortArgs' => ['-t', ' ', '-r', '+0', '-1'],
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
'num' => 17,
@@ -1291,6 +1301,8 @@ store c into ':OUTPATH:';\,
# This SQL query isn't right, because you can't group by name and have
# age in the output, like you can in Pig Latin
'sql' => "select name, age, gpa, max(age) from studenttab10k group by name, age, gpa order by age;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
# test to make sure the weighted range patitioning
@@ -1438,14 +1450,13 @@ select name, age from votertab10k;",
'name' => 'Bincond',
'tests' => [
{
- # Results doctored, if you change this query you need to copy the
- # expected results into test/nightly/benchmarks
- # Only until we do integer arithmetic with integers.
'num' => 1,
'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
b = foreach a generate name, (name matches 'yuri.*' ? age - 10 : (int)age);
store b into ':OUTPATH:';\,
'sql' => "select name, case when name like 'yuri%' then age - 10 else age end from studenttab10k;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
]
},
@@ -1749,7 +1760,6 @@ store b into ':OUTPATH:';\,
},
{
# NULL and cast
- # Results doctored
'num' => 2,
'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
b = foreach a generate (int)((int)gpa/((int)gpa - 1)) as norm_gpa:int;
@@ -1758,6 +1768,8 @@ store c into ':OUTPATH:';\,
'sql' => "select case when (cast(gpa as integer) - 1 = 0 or gpa is null) then 0 else cast ((cast(gpa as integer)/(cast(gpa as integer) - 1)) as integer) end from studentnulltab10k;",
# 'expected_err_regex' => "Encountered Warning DIVIDE_BY_ZERO 2387 time.*",
# Driver does currently not support both 'sql' and 'expected_...' verification directives.
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
# arithmetic operators and SIZE for int, double and size and concat operators for chararrays
@@ -1791,10 +1803,9 @@ store b into ':OUTPATH:';\,
},
{
- # sum, min, max, avg for chararray, int and double (declared)
- # WILL NEED DOCTORED RESULTS DUE TO PRECISION DIFFERENCES AND
- # BECAUSE IN PIG CURRENTLY COUNT INCLUDES NULLS - THIS AFFECTS
- # AVG
+ # will need to test against previous version of pig
+ # because in pig currently count includes nulls - this affects
+ # avg
'num' => 6,
'floatpostprocess' => 1,
'delimiter' => ' ',
@@ -1803,11 +1814,12 @@ b = group a ALL;
c = foreach b generate SUM(a.age), MIN(a.age), MAX(a.age), AVG(a.age), MIN(a.name), MAX(a.name), SUM(a.gpa), MIN(a.gpa), MAX(a.gpa), AVG(a.gpa);
store c into ':OUTPATH:';\,
'sql' => "select sum(age), min(age), max(age), avg(age), min(name), max(name), sum(gpa), min(gpa), max(gpa), avg(gpa) from studentnulltab10k;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
# sum, min, max, avg for long and float (declared)
- # WILL NEED DOCTORED RESULTS DUE TO PRECISION DIFFERENCES
'num' => 7,
'floatpostprocess' => 1,
'delimiter' => ' ',
@@ -1816,7 +1828,8 @@ b = group a ALL;
c = foreach b generate SUM(a.age), MIN(a.age), MAX(a.age), AVG(a.age), SUM(a.gpa), MIN(a.gpa), MAX(a.gpa), AVG(a.gpa);
store c into ':OUTPATH:';\,
'sql' => "select sum(age), min(age), max(age), avg(age), sum(gpa), min(gpa), max(gpa), avg(gpa) from studentnulltab10k;",
-
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
# Explicit casts - arithmetic operators and SIZE for int, double and size and concat operators for chararrays
@@ -1847,6 +1860,8 @@ c = group b ALL;
d = foreach c generate COUNT(b);
store d into ':OUTPATH:';\,
'sql' => "select count(*) from studentnulltab10k where name is null and age is not null and gpa is null;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
# Filter is not null for chararray and double and is null for int
@@ -1867,6 +1882,8 @@ c = group b ALL;
d = foreach c generate COUNT(b);
store d into ':OUTPATH:';\,
'sql' => "select count(*) from studentnulltab10k where name is null and age is not null and gpa is null;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
# Filter is not null for bytearray and float and is null for long
@@ -1930,16 +1947,6 @@ b = foreach a generate 0.123456789123456
store b into ':OUTPATH:';\,
'sql' => "select 0.123456789123456+0.123456789123456 from studentnulltab10k;",
},
-# {
- # argument construction for agg functions
-# expressions under agg are currently not supported
-# 'num' => 19,
-# 'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
-#b = group a by name;
-#c = foreach b generate group, SUM(a.age*a.gpa);
-#store c into ':OUTPATH:';\,
-# 'sql' => "select name, sum(age*gpa) from studentnulltab10k group by name;",
-# },
{
# order by string
'num' => 20,
@@ -2131,6 +2138,8 @@ store f into ':OUTPATH:';\,
'sql' => "select a.name, a.age, a.gpa, b.name, b.age, b.registration, b.contributions
from studenttab10k as a full outer join votertab10k as b using(age)
where a.age < 20 and b.age < 20;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
'num' => 36,
@@ -2146,28 +2155,30 @@ store f into ':OUTPATH:';\,
'sql' => "select a.name, a.age, a.gpa, b.name, b.age, b.registration, b.contributions
from studenttab10k as a full outer join votertab10k as b using(age)
where a.age < 20 and b.age < 20;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
# NULL and cast
- # Results doctored
'num' => 37,
'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
b = foreach a generate (int)((int)gpa/((int)gpa - 1)) as norm_gpa:int;
c = foreach b generate (norm_gpa is not null? norm_gpa: 0);
store c into ':OUTPATH:';\,
'sql' => "select case when (cast(gpa as integer) - 1 = 0 or gpa is null) then 0 else cast ((cast(gpa as integer)/(cast(gpa as integer) - 1)) as integer) end from studentnulltab10k;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
-# {
-# Commenting out test case till issue with negative PIG-522 is resolved
+ {
# constants
-# 'num' => 38,
-# 'floatpostprocess' => 1,
-# 'delimiter' => ' ',
-# 'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
-#b = foreach a generate -(age + 1 + 0.2f + 253645L), -(gpa+1);
-#store b into ':OUTPATH:';\,
-# 'sql' => "select -(age + 1 + cast(0.2 as float) + 253645), -(gpa + 1) from studentnulltab10k;"
-# },
+ 'num' => 38,
+ 'floatpostprocess' => 1,
+ 'delimiter' => ' ',
+ 'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name:chararray, age:int, gpa:double);
+b = foreach a generate -(age + 1 + 0.2f + 253645L), -(gpa+1);
+store b into ':OUTPATH:';\,
+ 'sql' => "select -(age + 1 + cast(0.2 as float) + 253645), -(gpa + 1) from studentnulltab10k;"
+ },
]
},
{
@@ -2175,8 +2186,8 @@ store c into ':OUTPATH:';\,
'floatpostprocess' => 1,
'tests' => [
{
- # IMPORTANT NOTE: results need to be stored in benchmarks since
- # we order nulls as < any value
+ # IMPORTANT NOTE: results need to be compared against old pig
+ # since we order nulls as < any value
# while postgreSQL does the opposite
'num' => 1,
'delimiter' => ' ',
@@ -2187,6 +2198,8 @@ d = limit c 100;
store d into ':OUTPATH:';\,
'sql' => "select * from studentnulltab10k where name > 'a' order by name, age limit 100;",
'sortArgs' => ['-t', ' ', '+0', '-1'],
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
'num' => 2,
@@ -2197,6 +2210,8 @@ c = limit b 100;
store c into ':OUTPATH:';\,
'sql' => "select * from studentnulltab10k order by name, age limit 100;",
'sortArgs' => ['-t', ' ', '+0', '-1'],
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
# Make sure that limit higher than number of rows doesn't mess stuff up
@@ -2217,6 +2232,8 @@ b = distinct a;
c = limit b 100;
store c into ':OUTPATH:';\,
'sql' => "select distinct * from studentnulltab10k limit 100;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
'num' => 5,
@@ -2232,6 +2249,8 @@ store d into ':OUTPATH:';\,
union all
select name, age from votertab10k
limit 100;;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
'num' => 6,
@@ -2243,6 +2262,8 @@ D = group C by name;
E = foreach D generate group, COUNT(C);
store E into ':OUTPATH:';\,
'sql' => "select name, count(*) from (select * from studenttab10k limit 40) where age = 40 group by name;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
'num' => 7,
@@ -2351,6 +2372,8 @@ store d into ':OUTPATH:';\,
from (select distinct * from studenttab10k where age > 50) as A
join (select * from studenttab10k where name < 'm') as B using (name)
group by A.name;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
{
'num' => 9,
@@ -2365,6 +2388,8 @@ store d into ':OUTPATH:';\,
from (select distinct * from studenttab10k where age > 50) as A
join (select * from studenttab10k where name < 'm') as B using (name)
group by name;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
}
]
},
@@ -3093,6 +3118,8 @@ from studenttab10k as a full outer join
'name' => 'SecondarySort',
'floatpostprocess' => 1,
'delimiter' => ' ',
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
'tests' => [
{
# simple order by
@@ -3403,6 +3430,10 @@ store e into ':OUTPATH:';\,
{
'name' => 'Native',
'delimiter' => ' ',
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
+ 'ignore' => q/These tests need reworked. They assume the existence
+ of files to be removed, but fail if the files aren't there./,
'tests' => [
{
# test common
@@ -3450,13 +3481,15 @@ store b into ':OUTPATH:';\,
{
# test group
'num' => 1,
- 'exectype' => 'mapred', # since this join will run out of memory in local mode
+ 'exectype' => 'mapred', # since this join will run out of memory in local mode
'pig' => q\register :FUNCPATH:/testudf.jar;
a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age:int, gpa);
b = group a by age PARTITION BY org.apache.pig.test.utils.SimpleCustomPartitioner2 parallel 2;
c = foreach b generate group, COUNT(a);
store c into ':OUTPATH:';\,
'sql' => "select age, COUNT(*) from studenttab10k group by age;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
]
},
@@ -3620,6 +3653,8 @@ SELECT id,
FROM numbers
ORDER BY id;
",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
}, {
# 'num' => 6,
@@ -3823,6 +3858,8 @@ from studenttab10k as a join votertab10k
where a.age < 20
and b.age < 20
and a.age < ( select age from studenttab10k where age < 20 order by name limit 1 );",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
},
]
@@ -3839,7 +3876,7 @@ and a.age < ( select age from studenttab
# TEST : resulting schema for various projects using a combination of TOBAG/TOTUPLE and standard projections
# TEST : resulting schema for various projects using a combination of TOBAG/TOTUPLE using AS clause
- 'num' => 1
+ 'num' => 1
,'pig' => q?
A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
B = limit A 10;
@@ -3906,7 +3943,7 @@ describe G;
# TEST : TOBAG/TOTUPLE with positional parameters
# TEST : various projects using a combination of TOBAG/TOTUPLE and standard projections
# TEST : various projects using a combination of TOBAG/TOTUPLE using AS clause
- 'num' => 3
+ 'num' => 3
,'pig' => q?
A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
B = limit A 10;
@@ -3946,11 +3983,13 @@ store T4 into ':OUTPATH:.8';
select name, age, gpa from studenttab10k;
select name, age, gpa from studenttab10k;
select name, age, gpa from studenttab10k;
- select name, age, gpa from studenttab10k;"
+ select name, age, gpa from studenttab10k;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
}, {
# TEST : cast for TOTUPLE/TOBAG
- 'num' => 4
+ 'num' => 4
,'pig' => q?
A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
B= limit A 10;
@@ -3962,7 +4001,7 @@ store C into ':OUTPATH:';
}, {
# TEST : cast for TOTUPLE/TOBAG
- 'num' => 5
+ 'num' => 5
,'pig' => q?
A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
B= limit A 1;
@@ -3985,7 +4024,9 @@ select name from studenttab10k;
select name from studenttab10k;
select name from studenttab10k;
select name from studenttab10k;
-select name from studenttab10k;"
+select name from studenttab10k;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
}, {
@@ -4008,19 +4049,20 @@ store binb into ':OUTPATH:.4';
store tinb into ':OUTPATH:.5';
?
-#FIX Verifier
,'sql' => "
select name from studenttab10k;
select name from studenttab10k;
select name from studenttab10k;
select name from studenttab10k;
-select name from studenttab10k;"
+select name from studenttab10k;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
}, {
#TEST arithmetic operation in TOTUPLE and TOBAG
#TEST aggregate funcion - NOT IMPLEMENTED
#TEST tuple with 50+ items
#TEST with null
- 'num' => 7
+ 'num' => 7
,'pig' => q?
A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
B = limit A 10;
@@ -4051,9 +4093,9 @@ select name from studenttab10k;
select name from studenttab10k;
select name from studenttab10k;
select name from studenttab10k;
-select name from studenttab10k;"
-
-# NEED TO ADD DATA VERIFIER- checked thse they are correct Pig 0.8.0
+select name from studenttab10k;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
# ,'expected_out_regex' => 'R1: {org.apache.pig.builtin.totuple_id_13: (id: int,int,int,int),{int}}'
# ,'expected_out_regex' => 'R2: {org.apache.pig.builtin.totuple_id_20: (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double,int,int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double,int,int,intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double,int,int,intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double,int,int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double,int,int,intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,longnum: long,doublenum: double)}'
@@ -4064,7 +4106,7 @@ select name from studenttab10k;"
}, {
# TESST more TOTUPLE and TOBAG nested combinations
- 'num' => 8
+ 'num' => 8
,'pig' => q?
A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
B = limit A 10;
@@ -4084,7 +4126,9 @@ select name from studenttab10k;
select name from studenttab10k;
select name from studenttab10k;
select name from studenttab10k;
-select name from studenttab10k;"
+select name from studenttab10k;",
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
}, {
#TEST negative test case: out of bounds positional parameter
@@ -4114,8 +4158,10 @@ C = foreach B generate $0, $1, TOBAG($5,
},{
# Test Union using merge with Simple data types
- 'name' => 'MergeOperator'
- ,'tests' => [
+ 'name' => 'MergeOperator',
+ 'verify_with_pig' => 1,
+ 'verify_pig_version' => 'old',
+ 'tests' => [
{
# - identical schema and duplicate data
# Test Union using merge with Simple data types | numbers has all the numeric types
@@ -4205,7 +4251,8 @@ SELECT id, intnum5 FROM numbers WHERE id
# Test Union using merge where result set is empty | results of H |
},{
- 'num' => 3
+ 'num' => 3,
+ 'ignore' => 'Test aborts with typechecker error',
,'pig' => q\
A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as ( Aintnum1000: int,Mid: int,Mintnum5: int,Mintnum100: int,Aintnum: int,Mlongnum: long,Mfloatnum: float,Mdoublenum: double);
B = load ':INPATH:/types/numbers.txt' using PigStorage(':') as ( Bintnum1000: int,Mid: int,Mintnum5: int,Mintnum100: int,Bintnum: int,Mlongnum: long,Mfloatnum: float,Mdoublenum: double);
@@ -4252,7 +4299,8 @@ WHERE id > 999999;"
# Test Union using merge where data do not intersect| C and D both have exclusive data in intnum5
# Test Union using merge where schema intersect | A&B have intersecting schema
# Test Union using merge where schema do not intersect | results of U2 |
- 'num' => 4
+ 'num' => 4,
+ 'ignore' => 'Error message problem, not clear'
,'pig' => q\
A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
B = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
@@ -4418,7 +4466,8 @@ ORDER BY id, intnum5;"
# Test Union using merge with a relation that is empty
# Remove this once regex is fixed
- 'num' => 12
+ 'num' => 12,
+ 'ignore' => 'Error message has changed.',
,'pig' => q\
A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
B = filter A by intnum5 < 3 and id < 20;
Modified: pig/trunk/test/e2e/pig/tests/turing_jython.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/turing_jython.conf?rev=1136214&r1=1136213&r2=1136214&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/turing_jython.conf (original)
+++ pig/trunk/test/e2e/pig/tests/turing_jython.conf Wed Jun 15 21:54:50 2011
@@ -35,16 +35,8 @@ $cfg = {
'name' => 'Jython_Checkin',
'tests' => [
{
- 'num' => 1,
- 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
-store a into ':OUTPATH:';\,
- 'sql' => "select name, age, gpa from studenttab10k;",
- 'floatpostprocess' => 1,
- 'delimiter' => ' ',
- },
- {
'num' => 2,
- ,'pig' => q\#!/usr/bin/python
+ 'pig' => q\#!/usr/bin/python
from org.apache.pig.scripting import Pig
input1=':INPATH:/singlefile/studenttab10k'
@@ -5880,6 +5872,7 @@ select name,age from studenttab10k where
}, {
# 11.23 1 explain() on a complex query
'num' => 6
+ ,'ignore' => 'Expected output does not appear to be correct'
,'pig' => q\#!/usr/bin/python
from org.apache.pig.scripting import Pig
@@ -6060,6 +6053,7 @@ result = P.bind({'in1':input1, 'in2':inp
}, {
#12.16 1 python for loop invoking pig script
'num' => 12
+ ,'ignore' => 'Trys to write everything to the same file, I think this is a test tool error not a Pig error.'
,'pig' => q\#!/usr/bin/python
from org.apache.pig.scripting import Pig
@@ -6383,8 +6377,12 @@ else:
\
,'sql' => "select name from textdoc;"
- ,'floatpostprocess' => 1
- ,'delimiter' => ' '
+ ,'verify_with_pig' => 1
+ ,'verify_pig_script' => q\register ':FUNCPATH:/testudf.jar';
+A = load ':INPATH:/singlefile/textdoc' using TextLoader();
+B = foreach A generate TOKENIZE((chararray)$0);
+store B into ':OUTPATH:' using org.apache.pig.test.udf.storefunc.StringStore();\,
+
},{
# 12.2 import python modules
@@ -6862,7 +6860,7 @@ else:
\
,'rc' => 6
- ,'expected_err_regex' => "ERROR 1000: Error during parsing. Invalid scalar projection: B"
+ ,'expected_err_regex' => "Invalid scalar projection: B"
}, {