You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2011/08/24 01:47:27 UTC
svn commit: r1160938 [1/2] - in /pig/trunk: CHANGES.txt
test/e2e/pig/build.xml test/e2e/pig/drivers/TestDriverPig.pm
test/e2e/pig/tests/bigdata.conf test/e2e/pig/tests/grunt.conf
test/e2e/pig/tests/turing_jython.conf
Author: gates
Date: Tue Aug 23 23:47:27 2011
New Revision: 1160938
URL: http://svn.apache.org/viewvc?rev=1160938&view=rev
Log:
PIG-2235: Several files in e2e tests aren't being run
Modified:
pig/trunk/CHANGES.txt
pig/trunk/test/e2e/pig/build.xml
pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm
pig/trunk/test/e2e/pig/tests/bigdata.conf
pig/trunk/test/e2e/pig/tests/grunt.conf
pig/trunk/test/e2e/pig/tests/turing_jython.conf
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1160938&r1=1160937&r2=1160938&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Aug 23 23:47:27 2011
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-2235: Several files in e2e tests aren't being run (gates)
+
PIG-2196: Test harness should be independent of Pig (hashutosh) -- Missed few
changes in last commit.
Modified: pig/trunk/test/e2e/pig/build.xml
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/build.xml?rev=1160938&r1=1160937&r2=1160938&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/build.xml (original)
+++ pig/trunk/test/e2e/pig/build.xml Tue Aug 23 23:47:27 2011
@@ -76,8 +76,6 @@
<copy todir="${tar.dir}/tests">
<fileset dir="${test.src}">
- <exclude name="bigdata.conf"/>
- <exclude name="grunt.conf"/>
<exclude name="streaming_local.conf"/>
</fileset>
</copy>
@@ -174,6 +172,8 @@
<arg value="${test.location}/tests/nightly.conf"/>
<arg value="${test.location}/tests/streaming.conf"/>
<arg value="${test.location}/tests/turing_jython.conf"/>
+ <arg value="${test.location}/tests/bigdata.conf"/>
+ <arg value="${test.location}/tests/grunt.conf"/>
</exec>
</target>
Modified: pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm?rev=1160938&r1=1160937&r2=1160938&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm (original)
+++ pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm Tue Aug 23 23:47:27 2011
@@ -26,6 +26,7 @@ use IPC::Run; # don't do qw(run), it scr
use Digest::MD5 qw(md5_hex);
use Util;
use File::Path;
+use Cwd;
use strict;
use English;
@@ -189,6 +190,20 @@ sub globalSetup
IPC::Run::run(['mkdir', '-p', $globalHash->{'localpath'}], \undef, $log, $log) or
die "Cannot create localpath directory " . $globalHash->{'localpath'} .
" " . "$ERRNO\n";
+
+ # Create the temporary directory
+ IPC::Run::run(['mkdir', '-p', $globalHash->{'tmpPath'}], \undef, $log, $log) or
+ die "Cannot create temporary directory " . $globalHash->{'tmpPath'} .
+ " " . "$ERRNO\n";
+}
+
+sub globalCleanup
+{
+ my ($self, $globalHash, $log) = @_;
+
+ IPC::Run::run(['rm', '-rf', $globalHash->{'tmpPath'}], \undef, $log, $log) or
+ warn "Cannot remove temporary directory " . $globalHash->{'tmpPath'} .
+ " " . "$ERRNO\n";
}
@@ -763,7 +778,7 @@ print $log "testResult: $testResult test
print $log "Test output checksum does not match benchmark checksum\n";
print $log "Test checksum = <$testChksm>\n";
print $log "Expected checksum = <$benchmarkChksm>\n";
- print $log "RESULTS DIFFER: vimdiff $testOutput $benchmarkOutput\n";
+ print $log "RESULTS DIFFER: vimdiff " . cwd . "/$testOutput " . cwd . "/$benchmarkOutput\n";
} else {
$result = 1;
}
Modified: pig/trunk/test/e2e/pig/tests/bigdata.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/bigdata.conf?rev=1160938&r1=1160937&r2=1160938&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/bigdata.conf (original)
+++ pig/trunk/test/e2e/pig/tests/bigdata.conf Tue Aug 23 23:47:27 2011
@@ -24,7 +24,7 @@
$cfg = {
'driver' => 'Pig',
- 'nummachines' => 10,
+ 'execonly' => 'mapred',
'groups' => [
{
@@ -32,6 +32,8 @@ $cfg = {
'tests' => [
{
'num' => 1,
+ ,'floatpostprocess' => 1
+ ,'delimiter' => ' ',
'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
c = filter a by age < '50';
@@ -42,37 +44,21 @@ g = group f by registration parallel 20;
h = foreach g generate group, SUM(f.d::contributions);
i = order h by $1, $0 parallel 20;
store i into ':OUTPATH:';\,
- 'sql' => "select registration, sum(contributions)
-from studenttab20m as a full outer join votertab10k as b using (name, age)
-where a.age < 50 and b.age < 50
-group by registration
-order by 2, 1;",
- 'floatpostprocess' => 1,
- 'delimiter' => ' ',
- },
-# {
-# 'num' => 2,
-# 'sortBenchmark' => 1,
-# 'sortResults' => 1,
-# 'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
-#a1 = filter a by age < '50';
-#b = group a1 by (name, age);
-#c = foreach b generate group as g, AVG(a1.gpa);
-#d = filter c by $1 > '3.0';
-#d1 = foreach d generate g.$0 as name, g.$1 as age, $1 as gpa;
-#e = group d1 by name;
-#f = foreach e generate group, AVG(d1.age);
-#store f into ':OUTPATH:';\,
-# 'sql' => "select name, avg(age)
-#from (select name, age, avg(gpa) from studenttab20m where age < 50 group by name, age having avg(gpa) > 3.0) as T
-#group by name;",
-# 'floatpostprocess' => 1,
-# 'delimiter' => ' ',
-# },
+ },
+ {
+ 'num' => 2,
+ 'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
+a1 = filter a by age < '50';
+b = group a1 by (name, age);
+c = foreach b generate group as g, AVG(a1.gpa);
+d = filter c by $1 > 3.0;
+d1 = foreach d generate g.$0 as name, g.$1 as age, $1 as gpa;
+e = group d1 by name;
+f = foreach e generate group, AVG(d1.age);
+store f into ':OUTPATH:';\,
+ },
{
'num' => 3,
- 'sortBenchmark' => 1,
- 'sortResults' => 1,
'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
c = foreach a generate name;
@@ -81,70 +67,46 @@ e = filter c by name matches '.*allen$';
f = union d, e;
g = distinct f parallel 20;
store g into ':OUTPATH:';\,
- 'sql' => q\select name from studenttab20m where name like '%allen'
-union
-select name from votertab10k;\,
},
{
'num' => 4,
- 'sortBenchmark' => 1,
- 'sortResults' => 1,
'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
b = group a all parallel 20;
c = foreach b generate COUNT(a.$0);
store c into ':OUTPATH:';\,
- 'sql' => "select count(*) from studenttab20m;",
},
{
'num' => 5,
- 'sortBenchmark' => 1,
- 'sortResults' => 1,
'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
b = group a by name parallel 20;
c = foreach b generate group, COUNT($1);
store c into ':OUTPATH:';\,
- 'sql' => "select name, count(*) from studenttab20m group by name;",
},
{
'num' => 6,
- 'sortBenchmark' => 1,
- 'sortResults' => 1,
'pig' => q\
define cmd `perl PigStreaming.pl` ship(':SCRIPTHOMEPATH:/PigStreaming.pl') stderr('CMD' limit 3);
a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
b = stream a through cmd as (n, a, g);
c = foreach b generate n, a;
store c into ':OUTPATH:';\,
- 'sql' => "select name, age from studenttab20m;",
},
{
'num' => 7,
- 'sortBenchmark' => 1,
- 'sortResults' => 1,
'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
b = order a by name parallel 20;
store b into ':OUTPATH:';\,
- 'sql' => "select * from studenttab20m;",
'sortArgs' => ['-t', ' ', '+0', '-1'],
- 'floatpostprocess' => 1,
- 'delimiter' => ' ',
},
{
'num' => 8,
- 'sortBenchmark' => 1,
- 'sortResults' => 1,
'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name:chararray, age:int, gpa:double);
b = order a by name, age desc parallel 20;
store b into ':OUTPATH:';\,
- 'sql' => "select * from studenttab20m;",
'sortArgs' => ['-t', ' ', '+0', '-1', '+1rn', '-2'],
- 'floatpostprocess' => 1,
- 'delimiter' => ' ',
},
{
'num' => 9,
- 'sortBenchmark' => 1,
- 'sortResults' => 1,
'pig' => q\A = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
B = filter A by age > 20;
C = group B by name;
@@ -152,18 +114,8 @@ D = foreach C generate group, COUNT(B) P
E = order D by $0 PARALLEL 16;
F = limit E 10;
store F into ':OUTPATH:';\,
- 'sql' => "select name, count(*) from studenttab20m where age > 20 group by name order by name limit 10;",
'sortArgs' => ['-t', ' ', '+0', '-1'],
},
- {
- 'num' => 10,
- 'floatpostprocess' => 1,
- 'delimiter' => ' ',
- 'pig' =>q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
-store a into ':OUTPATH:';\,
- 'sql' => "select name, age, gpa from studenttab20m;"
- }
-
]
},
Modified: pig/trunk/test/e2e/pig/tests/grunt.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/grunt.conf?rev=1160938&r1=1160937&r2=1160938&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/grunt.conf (original)
+++ pig/trunk/test/e2e/pig/tests/grunt.conf Tue Aug 23 23:47:27 2011
@@ -36,448 +36,89 @@ $cfg = {
'name' => 'Grunt',
'tests' => [
{
-# TEST Simple Test using ls
- 'num' => 100,
- 'pig' => q\
-ls :INPATH:
-sh ls
-sh echo hello
-sh ls -l :PIGHARNESS:
-sh echo hello2
-\,
- ,'expected_out_regex' => "hdfs:.*singlefile"
- ,'expected_out_regex' => "hello"
- ,'expected_out_regex' => "hello2"
- ,'expected_out_regex' => "libexec"
- ,'rc' => 0
-
- },{
-# TEST Extensive Test using ls
'num' => 1,
- 'pig' => q\
-sh ls -a :PIGHARNESS:/in/sql/address.txt
-sh ls -l :PIGHARNESS:/in/sql/dates.txt
-sh ls :PIGHARNESS:
-sh bash -c 'ls :PIGHARNESS:/in/sql/student*'
-sh bash -c 'ls -l :PIGHARNESS:/in/sql/c*.txt'
-sh ls /bin
-ls
-\,
- ,'expected_out_regex' => "address.txt"
- ,'expected_out_regex' => "dates.txt"
- ,'expected_out_regex' => "libexec"
- ,'expected_out_regex' => "-r.-r.-r--"
- ,'expected_out_regex' => "student"
- ,'expected_out_regex' => "customer"
- ,'expected_out_regex' => "rmdir"
- ,'rc' => 0
-
+ 'pig' => "sh ls :PIGHARNESS:",
+ 'expected_out_regex' => "libexec",
+ 'rc' => 0
},{
-# TEST pwd
'num' => 2,
- 'pig' => q\
-pwd
-sh pwd
-\,
- ,'expected_out_regex' => "hdfs:"
- ,'rc' => 0
-
- },{
-# TEST for cat
- 'num' => 3,
- 'pig' => q\
-sh cat :PIGHARNESS:/in/sql/numbers.txt
-\,
- ,'expected_out_regex' => "5000:5000:2:906:5060:5060L:5000.2f:5000.2"
- ,'rc' => 0
+ 'pig' => "pwd",
+ 'execonly' => 'mapred', # don't have a clue what their cwd will be for local mode
+ 'expected_out_regex' => "hdfs:",
+ 'rc' => 0
},{
-# TEST for pipe using cat | wc
-#BROKEN should be fixed with nightly - pipe fixed
'num' => 4,
- 'pig' => q\
-sh bash -c 'cat :PIGHARNESS:/in/sql/numbers.txt | wc'
-\,
- ,'expected_out_regex' => "5000.*5000.*199455"
+ 'pig' => "ls :INPATH:",
+ ,'expected_out_regex' => ".*singlefile"
,'rc' => 0
-# TEST for cd
- },{
- 'num' => 5,
- 'pig' => q\
-sh bash -c cd :LOCALINPATH:
-\,
- ,'rc' => 0
-
-# TEST for touch
},{
'num' => 6,
'pig' => q\
-sh mkdir -p :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_6.test
-sh touch :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_6.test/newfile.txt
-sh ls -l :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_6.test/newfile.txt
-\,
- ,'expected_out_regex' => "Grunt_6.test/newfile.txt"
- ,'rc' => 0
- },{
-# TEST for rm
- 'num' => 7,
- 'pig' => q\
-sh mkdir -p :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_7.test
-sh touch :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_7.test/newfile.txt
-sh rm -fv :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_7.test/newfile.txt
-\,
- ,'expected_out_regex' => "removed.*Grunt_7.test/newfile.txt"
- ,'rc' => 0
- },{
-# TEST for mkdir
- 'num' => 8,
- 'pig' => q\
-sh mkdir -p :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_8.test/newdir
-sh touch :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_8.test/newdir/newfile.txt
-sh ls -l :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_8.test/newdir/newfile.txt
-\,
- ,'expected_out_regex' => "-r.-r.*newfile.txt"
- ,'rc' => 0
- },{
-# TEST for mkdir
-# TEST for chmod 775 file
- 'num' => 9,
- 'pig' => q\
-sh mkdir -p :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_9.test
-sh touch :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_9.test/newfile.txt
-sh chmod -v 755 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_9.test/newfile.txt
-\,
- ,'expected_out_regex' => "changed to 0755"
- ,'rc' => 0
- },{
- 'num' => 10,
- 'pig' => q\
-sh mkdir -vp :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_10.test/newdir
-sh rm -Rfv :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_10.test/newdir
-\,
- ,'expected_out_regex' => "mkdir: created directory.*newdir"
- ,'expected_out_regex' => "removed directory.*newdir"
- ,'rc' => 0
- },{
-# TEST for diff at directory level
- 'num' => 11,
- 'pig' => q\
-A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
-B = limit A 5;
-C = limit A 10;
-store B into ':OUTPATH:/B.out';
-store C into ':OUTPATH:/C.out';
-copyToLocal :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_11.tmp
-sh bash -c 'diff :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_11.tmp/B.out/part-r-00000 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_11.tmp/C.out/part-r-00000'
-\,
- ,'expected_out_regex' => "1000.*9.*1.*904.*5018.*5018.*904.501.*904.501"
- ,'rc' => 0
- },{
-# TEST for diff two file where one is empty file
- 'num' => 12,
- 'pig' => q\
-A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
-B = limit A 10;
-Empty = filter A by id > 99999;
-store B into ':OUTPATH:/B.out';
-store Empty into ':OUTPATH:/Empty.out';
-copyToLocal :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_12.test
-sh bash -c 'diff :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_12.test/B.out/part-r-00000 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_12.test/Empty.out/part-m-00000'
-\,
- ,'expected_out_regex'=>"1000.*10.*2.*900.*5051.*5051.*10.2.*10.2"
- ,'rc' => 0
-
- },{
-# TEST form chmod +x file
- 'num' => 13,
- 'pig' => q\
-A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
-B = limit A 10;
-store B into ':OUTPATH:';
-copyToLocal :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_13.test
-sh bash -c 'chmod -v -x :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_13.test/part-r-00000'
-\,
- ,'expected_out_regex' => "changed.*to.*0666.*rw-rw-rw-"
- ,'rc' => 0
-
- },{
-# TEST form chmod +775 file
- 'num' => 14,
- 'pig' => q\
-A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
-B = limit A 10;
-store B into ':OUTPATH:';
-copyToLocal :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_14.test
-sh bash -c 'chmod -v 775 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_14.test/part-r-00000'
-\,
- ,'expected_out_regex' => "changed.*to.*0775.*rwxrwxr-x"
-
- ,'rc' => 0
-
- },{
-# TEST for sort
-# TEST command "tail"
-# TEST multple unix commands in a single line
-#BROKEN NEEED UPDATE SHOULD WORK ON NIGHTLY
- 'num' => 15,
- 'pig' => q\
-A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
-B = limit A 10;
-store B into ':OUTPATH:';
-copyToLocal :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_15.test
-sh bash -c 'sort :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_15.test/part-r-00000 > :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_15.test/out.sorted'
-sh bash -c 'tail -n 2 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_15.test/out.sorted'
-\,
- ,'expected_out_regex' => "1000.*9.*1.*904.*5018.*5018.*904.501.*904.501"
-
- ,'rc' => 0
-
- },{
-# TEST for sort
-# TEST command "head"
-#BROKEN NEEED UPDATE SHOULD WORK ON NIGHTLY
- 'num' => 16,
- 'pig' => q\
-A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
-B = limit A 10;
-store B into ':OUTPATH:';
-copyToLocal :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_16.test
-sh bash -c 'sort :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_16.test/part-r-00000 > :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_16.test/out.sorted'
-sh bash -c 'head -n 2 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_16.test/out.sorted'
-\,
- ,'expected_out_regex' => "1000.*1.*3.*90.*5084.*5084.*8.3.*8.3"
-
- ,'rc' => 0
-
-
- },{
- 'num' => 17,
- 'pig' => q\
-sh whoami
-\,
- ,'expected_out_regex' => "hadoopqa"
- ,'rc' => 0
-
- },{
-# TEST for comment
-# TEST for wc
-# TEST for wc with pipe
-# COMMENTS NOT SUPPORTED
-# sh #THIS IS A COMMENT
-#sh cat :INPATH:/types/numbers.txt | wc
- 'num' => 18,
- 'pig' => q\
-copyToLocal :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_18.tmp/numbers.txt
-copyToLocal :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_18.tmp/numbers2.txt
-sh wc :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_18.tmp/numbers.txt
-sh wc -l :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_18.tmp/numbers2.txt
-\,
- ,'expected_out_regex' => "5000.*5000.*199455.*numbers.txt"
- ,'expected_out_regex' => "5000.*numbers2.txt"
- ,'rc' => 0
-
- },{
-# TEST for grep
-# TEST foringle quotes
-#BROKEN NEEED UPDATE SHOULD WORK ON NIGHTLY
-#May want t rewrite this so that we use pig to generate results to compare against
- 'num' => 19,
- 'pig' => q#
-copyToLocal :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_19.tmp/numbers.txt
-sh grep 503 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_19.tmp/numbers.txt
-sh grep :506*: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_19.tmp/numbers.txt
-sh bash -c 'grep \':501:\' :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_19.tmp/numbers.txt'
-#,
- ,'expected_out_regex' => "1000:501:5:90:5019:5019L:8.5f:8.5"
- ,'expected_out_regex' => "1000:503:3:90:5064:5064L:0.506490f:0.506490"
- ,'expected_out_regex' => "5000:4972:5:90:5066:5066L:5.9f:5.9"
- ,'rc' => 0
-
- },{
-# TEST for uniq
- 'num' => 20,
- 'pig' => q\
-copyToLocal :INPATH:/singlefile/studenttab10k :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_20.tmp/studenttab10k
-sh uniq :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_20.tmp/studenttab10k
-\,
- ,'expected_out_regex' => "gabriella.*brown.*60.*1.71"
- ,'rc' => 0
-
-# TEST for find
-# TEST for wildcard
-#sh find :INPATH: -name studenttab10k
- },{
- 'num' => 21,
- 'pig' => q\
-sh mkdir :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_21.tmp/singlefile
-copyToLocal :INPATH:/singlefile/studenttab10k :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_21.tmp/singlefile/studenttab10k
-sh find :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_21.tmp -name student*
-\,
- ,'expected_out_regex' => "studenttab10k"
- ,'rc' => 0
-
- },{
-# TEST for df
- 'num' => 22,
- 'pig' => q\
-sh df
-\,
- ,'expected_out_regex' => "Filesystem.*Used.*Available"
- ,'rc' => 0
-
- },{
-# TEST for du
- 'num' => 23,
- 'pig' => q\
-sh du :PIGHARNESS:/in
-\,
- ,'expected_out_regex' => "sql"
- ,'rc' => 0
-
- },{
-# TEST for comment - not supported
-#sh #IGNORE THIS
-# TEST for echo
- 'num' => 24,
- 'pig' => q\
-
-sh echo This little piggy went to market
-sh echo This little piggy stayed home
-sh echo This little piggy had roast beef
-sh echo This little piggy had none
-sh echo This little piggy went wee wee wee, all the way home
-\,
- ,'expected_out_regex' => "market"
- ,'expected_out_regex' => "stayed"
- ,'expected_out_regex' => "roast"
- ,'expected_out_regex' => "none"
- ,'expected_out_regex' => "all the way home"
- ,'rc' => 0
-
- },{
-# TEST for pipe
-#BROKEN NEEED UPDATE SHOULD WORK ON NIGHTLY
- 'num' => 25,
- 'pig' => q\
-
-sh bash -c 'echo This little piggy went to market | wc'
-sh bash -c 'echo This little piggy stayed home | wc'
-sh bash -c 'echo This little piggy had roast beef | wc'
-sh bash -c 'echo This little piggy had none | wc'
-sh bash -c 'echo This little piggy went wee wee wee, all the way home | wc'
-\,
- ,'expected_out_regex' => "1.*6.*33"
- ,'expected_out_regex' => "1.*5.*30"
- ,'expected_out_regex' => "1.*5.*27"
-
- ,'rc' => 0
-
-
- },{
-# TEST for environmet variable
- 'num' => 26,
- 'pig' => q\
-sh bash -c 'echo $PIG_HARNESS_ROOT'
-sh bash -c 'ls $PIG_HARNESS_ROOT'
-\,
- ,'expected_out_regex' => "libexec"
- ,'rc' => 0
-
-
-
-# TEST read file you don't have read rights for
- },{
- 'num' => 27,
- 'pig' => q\
sh touch /bin/bad
\,
,'expected_err_regex' => "Permission denied"
,'rc' => 0
- },{
- 'num' => 28,
- 'pig' => q\
-sh ls -l doesnotexist1
-sh ls -l doesnotexist2/bad
-sh ls -l :OUTPATH:/doesnotexist3.txt
-\,
- ,'expected_err_regex' => "doesnotexist1.*No such file or directory"
- ,'expected_err_regex' => "doesnotexist2/bad.*No such file or directory"
- ,'expected_err_regex' => "doesnotexist3.txt.*No such file or directory"
-
- ,'rc' => 0
-
- },{
-# TEST run a shell script
-# BUG-3990418 May result in "../../../../..: Permission denied" Error if the top
-# level directory does not have read permissions, this will not be fixed
-
- 'num' => 29,
- 'pig' => "
-sh $ENV{PIG_HARNESS_ROOT}/bin/run_fb.sh -h
-",
- ,'expected_out_regex' => "Usage.*run_fb.sh"
- ,'rc' => 0
- },{
-# TEST run awk script
-#BROKEN - removing this awk script for now
-# 'num' => 30,
-# 'pig' => q\
-#sh mkdir :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_30.tmp/types
-#copyToLocal :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_30.tmp/numbers.txt
-#sh bash -c 'awk '{ print }' :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_30.tmp/numbers.txt'
-#\,
-# ,'expected_err_regex' => "5000:5000:2:906:5060:5060L:5000.2f:5000.2"
-#
-# ,'rc' => 0
-# },{
-# TEST run sed script
- 'num' => 31,
- 'pig' => q\
-sh mkdir :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_31.tmp/types
-copyToLocal :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_31.tmp/types/numbers.txt
-sh bash -c 'sed -e "1d" :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_31.tmp/types/numbers.txt'
-\,
- ,'expected_err_regex' => "5000:5000:2:906:5060:5060L:5000.2f:5000.2"
- ,'rc' => 0
-
- },{
-#TEST run perl script
- 'num' => 32,
- 'pig' => q\
-sh mkdir :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_32.tmp/types
-copyToLocal :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_32.tmp/types/numbers.txt
-sh bash -c 'perl -p -e "s/:/+/g;" :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_32.tmp/types/numbers.txt'
-\,
- },{
-#TEST run perl script
-#BROKEN Causes a 2998 Unhandled Exception, check to see if fixed in nightly
- 'num' => 33,
- 'pig' => q\
-sh bash -c ls -l //grid/2/dev/pigqa/in/types; ls -l /grid/2/dev/pigqa/in/glob; ls -l //grid/2/dev/pigqa/in/types; ls -l /grid/2/dev/pigqa/in/glob; ls
-\,
-#sh bash -c 'ls -l //grid/2/dev/pigqa/in/types; ls -l /grid/2/dev/pigqa/in/glob; ls -l //grid/2/dev/pigqa/in/types; ls -l /grid/2/dev/pigqa/in/glob; ls'
- ,'expected_err_regex' => "address.txt"
-
- ,'rc' => 0
-
-#IGNORE THIS TEST ISN'T READY
-# },{
-# 'num' => 33,
-# 'pig' => q\
-#sh echo "export FOO=MyEnvVariable" > :OUTPATH:/env.txt
-#sh source :OUTPATH:/env.txt
-#sh source :OUTPATH:/env.txt; echo $FOO
-#\,
-# ,'expected_out_regex' => "E: {age: bytearray}"
-# ,'rc' => 0
-
- }
-
-#ADD
-#wildcards
-#Add sort with tab delimeter
+ },{
+ 'num' => 7,
+ 'pig' => "fs -ls :INPATH:",
+ 'expected_out_regex' => ".*singlefile",
+ 'rc' => 0
+ },{
+ 'num' => 8,
+ 'pig' => "cat :INPATH:/singlefile/studenttab10k",
+ 'expected_out_regex' => "gabriella brown\\t60\\t1.71",
+ 'rc' => 0
+ },{
+ 'num' => 9,
+ 'pig' => "cd :INPATH:
+ pwd",
+ 'expected_out_regex' => ".*data",
+ 'rc' => 0
+ },{
+ 'num' => 10,
+ 'pig' => "cp :INPATH:/singlefile/studenttab10k .
+ ls .",
+ 'expected_out_regex' => ".*studenttab10k",
+ 'rc' => 0
+ },{
+ 'num' => 11,
+ 'pig' => "cp :INPATH:/singlefile/studenttab10k ./fred
+ ls .",
+ 'expected_out_regex' => ".*fred",
+ 'rc' => 0
+ },{
+ 'num' => 12,
+ 'pig' => "cp :INPATH:/singlefile/studenttab10k ./jim
+ mv ./jim ./bob
+ ls .",
+ 'expected_out_regex' => ".*bob",
+ 'rc' => 0
+ },{
+ 'num' => 13,
+ 'pig' => "copyToLocal :INPATH:/singlefile/studenttab10k :TMP:
+ sh cat :TMP:/studenttab10k",
+ 'expected_out_regex' => "gabriella brown\\t60\\t1.71",
+ 'rc' => 0
+ },{
+ 'num' => 14,
+ 'pig' => "copyToLocal :INPATH:/singlefile/votertab10k :TMP:
+ copyFromLocal :TMP:/votertab10k ./joe
+ cat ./joe",
+ 'expected_out_regex' => "jessica xylophone\\t41\\tgreen\\t638.13",
+ 'rc' => 0
+ },{
+ 'num' => 15,
+ 'pig' => "rm fred bob joe",
+ 'not_expected_out_regex' => "joe",
+ 'rc' => 0
+ },{
+ 'num' => 16,
+ 'pig' => "rmf jill",
+ 'rc' => 0
+ }
]
},
],