You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2011/09/14 21:03:13 UTC
svn commit: r1170769 [2/7] - in /pig/branches/branch-0.9: ./
test/e2e/harness/ test/e2e/harness/test/conf/ test/e2e/harness/test/tests/
test/e2e/pig/ test/e2e/pig/conf/ test/e2e/pig/deployers/
test/e2e/pig/drivers/ test/e2e/pig/macros/ test/e2e/pig/tes...
Modified: pig/branches/branch-0.9/test/e2e/pig/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/e2e/pig/drivers/TestDriverPig.pm?rev=1170769&r1=1170768&r2=1170769&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/e2e/pig/drivers/TestDriverPig.pm (original)
+++ pig/branches/branch-0.9/test/e2e/pig/drivers/TestDriverPig.pm Wed Sep 14 19:03:12 2011
@@ -26,13 +26,14 @@ use IPC::Run; # don't do qw(run), it scr
use Digest::MD5 qw(md5_hex);
use Util;
use File::Path;
+use Cwd;
use strict;
use English;
our $className= "TestDriver";
our @ISA = "$className";
-our $ROOT = (defined $ENV{'PIG_HARNESS_ROOT'} ? $ENV{'PIG_HARNESS_ROOT'} : die "ERROR: You must set environment variable PIG_HARNESS_ROOT\n");
+our $ROOT = (defined $ENV{'HARNESS_ROOT'} ? $ENV{'HARNESS_ROOT'} : die "ERROR: You must set environment variable HARNESS_ROOT\n");
our $toolpath = "$ROOT/libexec/PigTest";
my $passedStr = 'passed';
@@ -48,47 +49,10 @@ sub new
my $class = ref($proto) || $proto;
my $self = $class->SUPER::new;
- $self->{'exectype'} = "mapred"; # till we know better (in globalSetup())!
- $self->{'ignore'} = "true"; # till we know better (in globalSetup())!
-
bless($self, $class);
return $self;
}
-###############################################################################
-# This method has been copied over from TestDriver to make changes to
-# support skipping tests which do not match current execution mode
-# or which were marked as 'ignore'
-#
-#
-# Static function, can be used by test_harness.pl
-# Print the results so far, given the testStatuses hash.
-# @param testStatuses - reference to hash of test status results.
-# @param log - reference to file handle to print results to.
-# @param prefix - A title to prefix to the results
-# @returns nothing.
-#
-sub printResults
-{
- my ($testStatuses, $log, $prefix) = @_;
-
- my ($pass, $fail, $abort, $depend, $skipped) = (0, 0, 0, 0, 0);
-
- foreach (keys(%$testStatuses)) {
- ($testStatuses->{$_} eq $passedStr) && $pass++;
- ($testStatuses->{$_} eq $failedStr) && $fail++;
- ($testStatuses->{$_} eq $abortedStr) && $abort++;
- ($testStatuses->{$_} eq $dependStr) && $depend++;
- ($testStatuses->{$_} eq $skippedStr) && $skipped++;
- }
-
- my $msg = "$prefix, PASSED: $pass FAILED: $fail SKIPPED: $skipped ABORTED: $abort " .
- "FAILED DEPENDENCY: $depend";
- print $log "$msg\n";
- print "$msg\r";
-}
-
-
sub replaceParameters
{
##!!! Move this to Util.pm
@@ -102,7 +66,7 @@ sub replaceParameters
$cmd =~ s/:OUTPATH:/$outfile/g;
# $ENV
- $cmd =~ s/:PIGHARNESS:/$ENV{PIG_HARNESS_ROOT}/g;
+ $cmd =~ s/:PIGHARNESS:/$ENV{HARNESS_ROOT}/g;
# $testCmd
$cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g;
@@ -114,9 +78,9 @@ sub replaceParameters
$cmd =~ s/:SCRIPTHOMEPATH:/$testCmd->{'scriptPath'}/g;
$cmd =~ s/:DBUSER:/$testCmd->{'dbuser'}/g;
$cmd =~ s/:DBNAME:/$testCmd->{'dbdb'}/g;
- $cmd =~ s/:LOCALINPATH:/$testCmd->{'localinpathbase'}/g;
- $cmd =~ s/:LOCALOUTPATH:/$testCmd->{'localoutpathbase'}/g;
- $cmd =~ s/:LOCALTESTPATH:/$testCmd->{'localpathbase'}/g;
+# $cmd =~ s/:LOCALINPATH:/$testCmd->{'localinpathbase'}/g;
+# $cmd =~ s/:LOCALOUTPATH:/$testCmd->{'localoutpathbase'}/g;
+# $cmd =~ s/:LOCALTESTPATH:/$testCmd->{'localpathbase'}/g;
$cmd =~ s/:BMPATH:/$testCmd->{'benchmarkPath'}/g;
$cmd =~ s/:TMP:/$testCmd->{'tmpPath'}/g;
@@ -147,25 +111,9 @@ sub globalSetup
$self->{'ignore'} = 'false';
}
- # if "-x local" was provided on the command line,
- # it implies pig should be run in "local" mode -so
- # change input and output paths
- if(defined($globalHash->{'x'}) && $globalHash->{'x'} eq 'local')
- {
- $self->{'exectype'} = "local";
- $globalHash->{'inpathbase'} = $globalHash->{'localinpathbase'};
- $globalHash->{'outpathbase'} = $globalHash->{'localoutpathbase'};
- }
$globalHash->{'outpath'} = $globalHash->{'outpathbase'} . "/" . $globalHash->{'runid'} . "/";
$globalHash->{'localpath'} = $globalHash->{'localpathbase'} . "/" . $globalHash->{'runid'} . "/";
- # extract the current zebra.jar file path from the classpath
- # and enter it in the hash for use in the substitution of :ZEBRAJAR:
- my $zebrajar = $globalHash->{'cp'};
- $zebrajar =~ s/zebra.jar.*/zebra.jar/;
- $zebrajar =~ s/.*://;
- $globalHash->{'zebrajar'} = $zebrajar;
-
# add libexec location to the path
if (defined($ENV{'PATH'})) {
$ENV{'PATH'} = $globalHash->{'scriptPath'} . ":" . $ENV{'PATH'};
@@ -174,45 +122,55 @@ sub globalSetup
$ENV{'PATH'} = $globalHash->{'scriptPath'};
}
- my @cmd = (Util::getBasePigCmd($globalHash), '-e', 'mkdir', $globalHash->{'outpath'});
-
- if($self->{'exectype'} eq "local")
- {
- @cmd = ('mkdir', '-p', $globalHash->{'outpath'});
- }
+ my @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'mkdir', $globalHash->{'outpath'});
- print $log join(" ", @cmd);
-
- if($self->{'exectype'} eq "mapred")
- {
- IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n";
- }
- else
- {
- IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create directory " . $globalHash->{'outpath'} . "\n";
- }
+ print $log "Going to run " . join(" ", @cmd) . "\n";
+ IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n";
IPC::Run::run(['mkdir', '-p', $globalHash->{'localpath'}], \undef, $log, $log) or
die "Cannot create localpath directory " . $globalHash->{'localpath'} .
" " . "$ERRNO\n";
+
+ # Create the temporary directory
+ IPC::Run::run(['mkdir', '-p', $globalHash->{'tmpPath'}], \undef, $log, $log) or
+ die "Cannot create temporary directory " . $globalHash->{'tmpPath'} .
+ " " . "$ERRNO\n";
+}
+
+sub globalCleanup
+{
+ my ($self, $globalHash, $log) = @_;
+
+ IPC::Run::run(['rm', '-rf', $globalHash->{'tmpPath'}], \undef, $log, $log) or
+ warn "Cannot remove temporary directory " . $globalHash->{'tmpPath'} .
+ " " . "$ERRNO\n";
}
sub runTest
{
- my ($self, $testCmd, $log, $copyResults) = @_;
+ my ($self, $testCmd, $log) = @_;
my $subName = (caller(0))[3];
+ # Check that we should run this test. If the current execution type
+ # doesn't match the execonly flag, then skip this one.
+ if ($self->wrongExecutionMode($testCmd)) {
+ print $log "Skipping test $testCmd->{'group'}" . "_" .
+ $testCmd->{'num'} . " since it is executed only in " .
+ $testCmd->{'execonly'} . " mode and we are executing in " .
+ $testCmd->{'exectype'} . " mode.\n";
+ my %result;
+ return \%result;
+ }
+
# Handle the various methods of running used in
# the original TestDrivers
if ( $testCmd->{'pig'} && $self->hasCommandLineVerifications( $testCmd, $log) ) {
- return $self->runPigCmdLine( $testCmd, $log, $copyResults );
+ return $self->runPigCmdLine( $testCmd, $log, 1);
} elsif( $testCmd->{'pig'} ){
- return $self->runPig( $testCmd, $log, $copyResults );
- #} elsif( $testCmd->{'pigsql'} ){
- # return $self->runPigSql( $testCmd, $log, $copyResults );
+ return $self->runPig( $testCmd, $log, 1);
} elsif( $testCmd->{'script'} ){
return $self->runScript( $testCmd, $log );
} else {
@@ -249,7 +207,7 @@ sub runPigCmdLine
close(FH);
# Build the command
- my @baseCmd = Util::getBasePigCmd($testCmd);
+ my @baseCmd = $self->getPigCmd($testCmd, $log);
my @cmd = @baseCmd;
# Add option -l giving location for secondary logs
@@ -341,6 +299,43 @@ sub runScript
}
+sub getPigCmd($$$)
+{
+ my ($self, $testCmd, $log) = @_;
+
+ my @pigCmd;
+
+ # set the PIG_CLASSPATH environment variable
+ my $pcp .= $testCmd->{'jythonjar'} if (defined($testCmd->{'jythonjar'}));
+ $pcp .= ":" . $testCmd->{'classpath'} if (defined($testCmd->{'classpath'}));
+ $pcp .= ":" . $testCmd->{'testconfigpath'} if ($testCmd->{'exectype'} ne "local");
+
+ # Set it in our current environment. It will get inherited by the IPC::Run
+ # command.
+ $ENV{'PIG_CLASSPATH'} = $pcp;
+
+ @pigCmd = ("$testCmd->{'pigpath'}/bin/pig");
+
+ if (defined($testCmd->{'additionaljars'})) {
+ push(@pigCmd, '-Dpig.additional.jars='.$testCmd->{'additionaljars'});
+ }
+
+ if ($testCmd->{'exectype'} eq "local") {
+ push(@{$testCmd->{'java_params'}}, "-Xmx1024m");
+ push(@pigCmd, ("-x", "local"));
+ }
+
+ if (defined($testCmd->{'java_params'})) {
+ $ENV{'PIG_OPTS'} = join(" ", @{$testCmd->{'java_params'}});
+ }
+
+
+ print $log "Returning Pig command " . join(" ", @pigCmd) . "\n";
+ print $log "With PIG_CLASSPATH set to " . $ENV{'PIG_CLASSPATH'} . "\n";
+ return @pigCmd;
+}
+
+
sub runPig
{
@@ -361,7 +356,7 @@ sub runPig
# Build the command
- my @baseCmd = Util::getBasePigCmd($testCmd);
+ my @baseCmd = $self->getPigCmd($testCmd, $log);
my @cmd = @baseCmd;
# Add option -l giving location for secondary logs
@@ -382,6 +377,7 @@ sub runPig
# Run the command
+ print $log "Setting PIG_CLASSPATH to $ENV{'PIG_CLASSPATH'}\n";
print $log "$0::$className::$subName INFO: Going to run pig command: @cmd\n";
IPC::Run::run(\@cmd, \undef, $log, $log) or
@@ -392,66 +388,35 @@ sub runPig
# Get results from the command locally
my $localoutfile;
my $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
- my @SQLQuery = @{$testCmd->{'queries'}}; # here only used to determine if single-guery of multi-query
+ my $stores = $self->countStores($testCmd);
- # mapreduce
- if($self->{'exectype'} eq "mapred")
- {
- # single query
- if ($#SQLQuery == 0) {
- if ($copyResults) {
- $result{'output'} = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log);
- $result{'originalOutput'} = "$localdir/out_original"; # populated by postProcessSingleOutputFile
- } else {
- $result{'output'} = "NO_COPY";
- }
- }
- # multi query
- else {
- my @outfiles = ();
- for (my $id = 1; $id <= ($#SQLQuery + 1); $id++) {
- $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out/$id";
- $localoutfile = $outfile . ".$id";
-
- # Copy result file out of hadoop
- my $testOut;
- if ($copyResults) {
- $testOut = $self->postProcessSingleOutputFile($localoutfile, $localdir, \@baseCmd, $testCmd, $log);
- } else {
- $testOut = "NO_COPY";
- }
- push(@outfiles, $testOut);
- }
- ##!!! originalOutputs not set! Needed?
- $result{'outputs'} = \@outfiles;
- }
- }
- # local mode
- else
- {
- # single query
- if ($#SQLQuery == 0) {
- $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".dir";
- mkdir $localdir;
+ # single query
+ if ($stores == 1) {
+ if ($copyResults) {
$result{'output'} = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log);
$result{'originalOutput'} = "$localdir/out_original"; # populated by postProcessSingleOutputFile
- }
- # multi query
- else {
- my @outfiles = ();
- for (my $id = 1; $id <= ($#SQLQuery + 1); $id++) {
- $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
- mkdir $localdir;
- $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out/$id";
- mkdir $localdir;
- $localoutfile = $outfile . ".$id";
+ } else {
+ $result{'output'} = "NO_COPY";
+ }
+ }
+ # multi query
+ else {
+ my @outfiles = ();
+ for (my $id = 1; $id <= ($stores); $id++) {
+ $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out/$id";
+ $localoutfile = $outfile . ".$id";
- my $testRes = $self->postProcessSingleOutputFile($localoutfile, $localdir, \@baseCmd, $testCmd, $log);
- push(@outfiles, $testRes);
+ # Copy result file out of hadoop
+ my $testOut;
+ if ($copyResults) {
+ $testOut = $self->postProcessSingleOutputFile($localoutfile, $localdir, \@baseCmd, $testCmd, $log);
+ } else {
+ $testOut = "NO_COPY";
}
- ##!!! originalOutputs not set!
- $result{'outputs'} = \@outfiles;
+ push(@outfiles, $testOut);
}
+ ##!!! originalOutputs not set! Needed?
+ $result{'outputs'} = \@outfiles;
}
# Compare doesn't get the testCmd hash, so I need to stuff the necessary
@@ -469,15 +434,12 @@ sub postProcessSingleOutputFile
my ($self, $outfile, $localdir, $baseCmd, $testCmd, $log) = @_;
my $subName = (caller(0))[3];
- # Copy to local if results on HDFS
- if ($self->{'exectype'} eq "mapred") {
- my @baseCmd = @{$baseCmd};
- my @copyCmd = @baseCmd;
- push(@copyCmd, ('-e', 'copyToLocal', $outfile, $localdir));
- print $log "$0::$className::$subName INFO: Going to run pig command: @copyCmd\n";
+ my @baseCmd = @{$baseCmd};
+ my @copyCmd = @baseCmd;
+ push(@copyCmd, ('-e', 'copyToLocal', $outfile, $localdir));
+ print $log "$0::$className::$subName INFO: Going to run pig command: @copyCmd\n";
- IPC::Run::run(\@copyCmd, \undef, $log, $log) or die "Cannot copy results from HDFS $outfile to $localdir\n";
- }
+ IPC::Run::run(\@copyCmd, \undef, $log, $log) or die "Cannot copy results from HDFS $outfile to $localdir\n";
# Sort the result if necessary. Keep the original output in one large file.
@@ -485,14 +447,13 @@ sub postProcessSingleOutputFile
# Build command to:
# 1. Combine part files
- my $fppCmd =
- ($self->{'exectype'} eq "mapred") ? "cat $localdir/map* $localdir/part* 2>/dev/null" :
- (-d $outfile ) ? "cat $outfile/part* 2>/dev/null" :
- "cat $outfile";
+ my $fppCmd = "cat $localdir/map* $localdir/part* 2>/dev/null";
# 2. Standardize float precision
- if (defined $testCmd->{'floatpostprocess'} && defined $testCmd->{'delimiter'}) {
- $fppCmd .= " | $toolpath/floatpostprocessor.pl '" . $testCmd->{'delimiter'} . "'";
+ if (defined $testCmd->{'floatpostprocess'} &&
+ defined $testCmd->{'delimiter'}) {
+ $fppCmd .= " | $toolpath/floatpostprocessor.pl '" .
+ $testCmd->{'delimiter'} . "'";
}
$fppCmd .= " > $localdir/out_original";
@@ -515,68 +476,31 @@ sub generateBenchmark
my %result;
- my @SQLQuery = @{$testCmd->{'queries'}};
-
- if ($#SQLQuery == 0) {
- my $outfile = $self->generateSingleSQLBenchmark($testCmd, $SQLQuery[0], undef, $log);
- $result{'output'} = $outfile;
- } else {
- my @outfiles = ();
- for (my $id = 0; $id < ($#SQLQuery + 1); $id++) {
- my $sql = $SQLQuery[$id];
- my $outfile = $self->generateSingleSQLBenchmark($testCmd, $sql, ($id+1), $log);
- push(@outfiles, $outfile);
- }
- $result{'outputs'} = \@outfiles;
- }
-
- return \%result;
-}
-
-sub generateSingleSQLBenchmark
-{
- my ($self, $testCmd, $sql, $id, $log) = @_;
-
- my $qmd5 = substr(md5_hex($testCmd->{'pig'}), 0, 5);
- my $outfile = $testCmd->{'benchmarkPath'} . "/" . $testCmd->{'group'} . "_" . $testCmd->{'num'};
- $outfile .= defined($id) ? ".$id" . ".expected." . $qmd5 : ".expected." . $qmd5;
-
- if (-e $outfile) {
- return $outfile;
- }
-
- my @cmd = ('psql', '-U', $testCmd->{'dbuser'}, '-d', $testCmd->{'dbdb'},
- '-c', $sql, '-t', '-A', '--pset', "fieldsep= ", '-o', $outfile);
-
- print $log "Running SQL command [" . join(" ", @cmd) . "\n";
- IPC::Run::run(\@cmd, \undef, $log, $log) or do {
- print $log "Sql command <" . $sql .
- " failed for >>$testCmd->{group}_$testCmd->{num}<<\n";
- unlink $outfile if ( -e $outfile );
-
- die "Sql command failed for >>$testCmd->{group}_$testCmd->{num}<<\n";
- };
-
- # Sort and postprocess the result if necessary
- my $sortCmd = "cat $outfile";
- if (defined $testCmd->{'floatpostprocess'} && defined $testCmd->{'delimiter'}) {
- $sortCmd .= " | $toolpath/floatpostprocessor.pl '" . $testCmd->{'delimiter'} . "' ";
- }
-
- # Use system not IPC run so that the '*' gets interpolated by the shell.
- $sortCmd .= " | sort";
- if (defined $testCmd->{'sortBenchmarkArgs'}) {
- $sortCmd .= " " . join(" ", @{$testCmd->{'sortBenchmarkArgs'}});
- }
- my $tmpfile = $outfile . ".tmp";
- $sortCmd .= " > $tmpfile";
- print $log "$sortCmd\n";
- system($sortCmd);
- unlink $outfile;
-
- IPC::Run::run ['mv', $tmpfile, $outfile];
-
- return $outfile;
+ # Check that we should run this test. If the current execution type
+ # doesn't match the execonly flag, then skip this one.
+ if ($self->wrongExecutionMode($testCmd)) {
+ return \%result;
+ }
+
+ if ($self->hasCommandLineVerifications($testCmd, $log)) {
+ # Do nothing, no benchmark to geneate
+ return \%result;
+ }
+
+ # If they specified an alternate Pig Latin script, use that on the current
+ # version. Otherwise use a previous version of Pig.
+ my %modifiedTestCmd = %{$testCmd};
+ if (defined $testCmd->{'verify_pig_script'}) {
+ $modifiedTestCmd{'pig'} = $testCmd->{'verify_pig_script'};
+ }
+ else {
+ # Change so we're looking at the old version of Pig
+ $modifiedTestCmd{'pigpath'} = $testCmd->{'oldpigpath'};
+ }
+ # Modify the test number so we don't run over the actual test output
+ # and logs
+ $modifiedTestCmd{'num'} = $testCmd->{'num'} . "_benchmark";
+ return $self->runPig(\%modifiedTestCmd, $log, 1);
}
sub hasCommandLineVerifications
@@ -598,6 +522,13 @@ sub compare
my ($self, $testResult, $benchmarkResult, $log, $testCmd) = @_;
my $subName = (caller(0))[3];
+ # Check that we should run this test. If the current execution type
+ # doesn't match the execonly flag, then skip this one.
+ if ($self->wrongExecutionMode($testCmd)) {
+ # Special magic value
+ return $self->{'wrong_execution_mode'};
+ }
+
# For now, if the test has
# - testCmd pig, and 'sql' for benchmark, then use comparePig, i.e. using benchmark
# - any verification directives formerly used by CmdLine or Script drivers (rc, regex on out and err...)
@@ -605,21 +536,14 @@ sub compare
# - testCmd script, then use compareScript
# - testCmd pig, and none of the above, then use comparePig
#
- # (The first condition included because nightly.conf actually had a test, Types_2, that
- # specified both 'sql' and 'expected_err_regex'.)
- #
# Later, should add ability to have same tests both verify with the 'script' directives,
# and do a benchmark compare, if it was a pig cmd. E.g. 'rc' could still be checked when
# doing the benchmark compare.
- if( defined $testCmd->{'sql'} ){
- return $self->comparePig ( $testResult, $benchmarkResult, $log, $testCmd);
- } elsif( $testCmd->{'script'} || $self->hasCommandLineVerifications( $testCmd, $log) ){
+ if ( $testCmd->{'script'} || $self->hasCommandLineVerifications( $testCmd, $log) ){
return $self->compareScript ( $testResult, $log, $testCmd);
} elsif( $testCmd->{'pig'} ){
return $self->comparePig ( $testResult, $benchmarkResult, $log, $testCmd);
- #} elsif( $testCmd->{'pigsql'} ){
- # return $self->comparePigSql ( $testResult, $benchmarkResult, $log, $testCmd);
} else {
# Should have been caught by runTest, still...
print $log "$0.$subName WARNING Did not find a testCmd that I know how to handle\n";
@@ -633,6 +557,7 @@ sub compareScript
my ($self, $testResult, $log, $testCmd) = @_;
my $subName = (caller(0))[3];
+
# IMPORTANT NOTES:
#
# If you are using a regex to compare stdout or stderr
@@ -663,7 +588,7 @@ sub compareScript
print $log "$0::$subName INFO Checking test stdout' " .
"as exact match against expected <$testCmd->{'expected_out'}>\n";
if ($testResult->{'stdout'} ne $testCmd->{'expected_out'}) {
- print $log "$0::$subName INFO Check failed: exact match of <$testCmd->{'expected_out'}> expected in stdout: $testResult->{'output'}\n";
+ print $log "$0::$subName INFO Check failed: exact match of <$testCmd->{'expected_out'}> expected in stdout: $testResult->{'stdout'}\n";
$result = 0;
}
}
@@ -672,7 +597,7 @@ sub compareScript
print $log "$0::$subName INFO Checking test stdout " .
"as NOT exact match against expected <$testCmd->{'expected_out'}>\n";
if ($testResult->{'stdout'} eq $testCmd->{'not_expected_out'}) {
- print $log "$0::$subName INFO Check failed: NON-match of <$testCmd->{'expected_out'}> expected to stdout: $testResult->{'output'}\n";
+ print $log "$0::$subName INFO Check failed: NON-match of <$testCmd->{'expected_out'}> expected to stdout: $testResult->{'stdout'}\n";
$result = 0;
}
}
@@ -681,7 +606,7 @@ sub compareScript
print $log "$0::$subName INFO Checking test stdout " .
"for regular expression <$testCmd->{'expected_out_regex'}>\n";
if ($testResult->{'stdout'} !~ $testCmd->{'expected_out_regex'}) {
- print $log "$0::$subName INFO Check failed: regex match of <$testCmd->{'expected_out_regex'}> expected in stdout: $testResult->{'output'}\n";
+ print $log "$0::$subName INFO Check failed: regex match of <$testCmd->{'expected_out_regex'}> expected in stdout: $testResult->{'stdout'}\n";
$result = 0;
}
}
@@ -690,7 +615,7 @@ sub compareScript
print $log "$0::$subName INFO Checking test stdout " .
"for NON-match of regular expression <$testCmd->{'not_expected_out_regex'}>\n";
if ($testResult->{'stdout'} =~ $testCmd->{'not_expected_out_regex'}) {
- print $log "$0::$subName INFO Check failed: regex NON-match of <$testCmd->{'not_expected_out_regex'}> expected in stdout: $testResult->{'output'}\n";
+ print $log "$0::$subName INFO Check failed: regex NON-match of <$testCmd->{'not_expected_out_regex'}> expected in stdout: $testResult->{'stdout'}\n";
$result = 0;
}
}
@@ -736,35 +661,25 @@ sub compareScript
}
-# sub comparePigSql
-# {
-# my ($self, $testResult, $benchmarkResult, $testCmd, $log) = @_;
-# my $subName = (caller(0))[3];
-#
-# my $result;
-# ##!!! STUB
-# }
-
-
sub comparePig
{
my ($self, $testResult, $benchmarkResult, $log, $testCmd) = @_;
my $subName = (caller(0))[3];
my $result;
- my @SQLQuery = @{$testCmd->{'queries'}};
+ my $stores = $self->countStores($testCmd);
- if ($#SQLQuery == 0) {
+ if ($stores == 1) {
$result = $self->compareSingleOutput($testResult, $testResult->{'output'},
$benchmarkResult->{'output'}, $log);
} else {
my $res = 0;
- for (my $id = 0; $id < ($#SQLQuery + 1); $id++) {
+ for (my $id = 0; $id < ($stores); $id++) {
my $testOutput = ($testResult->{'outputs'})->[$id];
my $benchmarkOutput = ($benchmarkResult->{'outputs'})->[$id];
$res += $self->compareSingleOutput($testResult, $testOutput,
$benchmarkOutput, $log);
- $result = ($res == ($#SQLQuery + 1)) ? 1 : 0;
+ $result = ($res == ($stores)) ? 1 : 0;
}
}
@@ -776,6 +691,8 @@ sub compareSingleOutput
{
my ($self, $testResult, $testOutput, $benchmarkOutput, $log) = @_;
+print $log "testResult: $testResult testOutput: $testOutput benchmarkOutput: $benchmarkOutput\n";
+
# cksum the the two files to see if they are the same
my ($testChksm, $benchmarkChksm);
IPC::Run::run((['cat', $testOutput], '|', ['cksum']), \$testChksm,
@@ -792,7 +709,7 @@ sub compareSingleOutput
print $log "Test output checksum does not match benchmark checksum\n";
print $log "Test checksum = <$testChksm>\n";
print $log "Expected checksum = <$benchmarkChksm>\n";
- print $log "RESULTS DIFFER: vimdiff $testOutput $benchmarkOutput\n";
+ print $log "RESULTS DIFFER: vimdiff " . cwd . "/$testOutput " . cwd . "/$benchmarkOutput\n";
} else {
$result = 1;
}
@@ -815,298 +732,43 @@ sub compareSingleOutput
return $result;
}
-###############################################################################
-# This method has been copied over from TestDriver to make changes to
-# support skipping tests which do not match current execution mode
-#
-#
-# Run all the tests in the configuration file.
-# @param testsToRun - reference to array of test groups and ids to run
-# @param testsToMatch - reference to array of test groups and ids to match.
-# If a test group_num matches any of these regular expressions it will be run.
-# @param cfg - reference to contents of cfg file
-# @param log - reference to a stream pointer for the logs
-# @param dbh - reference database connection
-# @param testStatuses- reference to hash of test statuses
-# @param confFile - config file name
-# @param startat - test to start at.
-# @returns nothing
-# failed.
+##############################################################################
+# Count the number of stores in a Pig Latin script, so we know how many files
+# we need to compare.
#
-sub run
+sub countStores($$)
{
- my ($self, $testsToRun, $testsToMatch, $cfg, $log, $dbh, $testStatuses,
- $confFile, $startat, $logname ) = @_;
- my $subName = (caller(0))[3];
-
- my $msg="";
- my $duration=0;
- my $totalDuration=0;
- my $groupDuration=0;
+ my ($self, $testCmd) = @_;
-
- my $sawstart = !(defined $startat);
- # Rather than make each driver handle our multi-level cfg, we'll flatten
- # the hashes into one for it.
- my %globalHash;
-
- my $runAll = ((scalar(@$testsToRun) == 0) && (scalar(@$testsToMatch) == 0));
-
- # Read the global keys
- foreach (keys(%$cfg)) {
- next if $_ eq 'groups';
- $globalHash{$_} = $cfg->{$_};
+ # Special work around for queries with more than one store that are not
+ # actually multiqueries.
+ if (defined $testCmd->{'notmq'}) {
+ return 1;
}
- # Do the global setup
- $self->globalSetup(\%globalHash, $log);
-
- my $report=0;
- my $properties= new Properties( 0, $globalHash{'propertiesFile'} );
+ my $count;
- # For the xml directory, use the default directory from the configuration file
- # unless the directory was specified in the command line
-
- my $xmlDir= $globalHash{'localxmlpathbase'} ."/run". $globalHash{'UID'};
- if ( $globalHash{'reportdir'} ) {
- $xmlDir = $globalHash{'reportdir'};
- }
-
- my %groupExecuted;
-
- foreach my $group (@{$cfg->{'groups'}}) {
- my %groupHash = %globalHash;
- $groupHash{'group'} = $group->{'name'};
-
- # Read the group keys
- foreach (keys(%$group)) {
- next if $_ eq 'tests';
- $groupHash{$_} = $group->{$_};
- }
-
- # Run each test
- foreach my $test (@{$group->{'tests'}}) {
- # Check if we're supposed to run this one or not.
- if (!$runAll) {
- # check if we are supposed to run this test or not.
- my $foundIt = 0;
- foreach (@$testsToRun) {
- if (/^$groupHash{'group'}(_[0-9]+)?$/) {
- if (not defined $1) {
- # In this case it's just the group name, so we'll
- # run every test in the group
- $foundIt = 1;
- last;
- } else {
- # maybe, it at least matches the group
- my $num = "_" . $test->{'num'};
- if ($num eq $1) {
- $foundIt = 1;
- last;
- }
- }
- }
- }
- foreach (@$testsToMatch) {
- my $protoName = $groupHash{'group'} . "_" . $test->{'num'};
- if ($protoName =~ /$_/) {
- if (not defined $1) {
- # In this case it's just the group name, so we'll
- # run every test in the group
- $foundIt = 1;
- last;
- } else {
- # maybe, it at least matches the group
- my $num = "_" . $test->{'num'};
- if ($num eq $1) {
- $foundIt = 1;
- last;
- }
- }
- }
- }
-
- next unless $foundIt;
- }
-
- # This is a test, so run it.
- my %testHash = %groupHash;
- foreach (keys(%$test)) {
- $testHash{$_} = $test->{$_};
- }
- my $testName = $testHash{'group'} . "_" . $testHash{'num'};
-
- if ( $groupExecuted{ $group->{'name'} }== 0 ){
- $groupExecuted{ $group->{'name'} }=1;
-
- mkpath( [ $xmlDir ] , 1, 0777) if ( ! -e $xmlDir );
-
- my $filename = $group->{'name'}.".xml";
- $report = new TestReport ( $properties, "$xmlDir/$filename" );
- $report->purge();
-
- }
-
- # Have we not reached the starting point yet?
- if (!$sawstart) {
- if ($testName eq $startat) {
- $sawstart = 1;
- } else {
- next;
- }
- }
-
- # Check that this test doesn't depend on an earlier test or tests
- # that failed, or that the test wasn't marked as "ignore".
- # Don't abort if that test wasn't run, just assume the
- # user knew what they were doing and set it up right.
- my $skipThisOne = 0;
- foreach (keys(%testHash)) {
- if (/^depends_on/ && defined($testStatuses->{$testHash{$_}}) &&
- $testStatuses->{$testHash{$_}} ne $passedStr) {
- print $log "Skipping test $testName, it depended on " .
- "$testHash{$_} which returned a status of " .
- "$testStatuses->{$testHash{$_}}\n";
- $testStatuses->{$testName} = $dependStr;
- $skipThisOne = 1;
- last;
- }
- # if the test is not applicable to current execution mode
- # ignore it
- if(/^exectype$/i && $testHash{$_} !~ /$self->{'exectype'}/i)
- {
- print $log "\nIGNORING test since running mode ($self->{'exectype'}) and exectype in test ($testHash{'exectype'}) do not match\n";
- $testStatuses->{$testName} = $skippedStr;
- $skipThisOne = 1;
- last;
- }
-
- # if the test is marked as 'ignore',
- # ignore it... unless option to ignore the ignore is in force
- if(/^ignore$/i && ($self->{'ignore'} eq 'true'))
- {
- print $log "$0::$className::$subName TEST IGNORED <$testName> at " . time . ". Message: $testHash{'ignore'}\n";
- $testStatuses->{$testName} = $skippedStr;
- $skipThisOne = 1;
- last;
- }
- }
-
- if ($skipThisOne) {
- printResults($testStatuses, $log, "Results so far");
- next;
- }
-
- # Check if output comparison should be skipped.
- my $dontCompareThisOne = 0; # true for tests with key 'noverify'
- my $copyResults = 1; # no need to copy output to local if noverify
- foreach (keys(%testHash)) {
-
- if(/^noverify$/i )
- {
- $dontCompareThisOne = 1;
- $copyResults = 0;
- last;
- }
- }
-
- print $log "Beginning test $testName at " . time . "\n";
- my %dbinfo = (
- 'testrun_id' => $testHash{'trid'},
- 'test_type' => $testHash{'driver'},
- #'test_file' => $testHash{'file'},
- 'test_file' => $confFile,
- 'test_group' => $testHash{'group'},
- 'test_num' => $testHash{'num'},
- );
- my $beginTime = time;
- my ($testResult, $benchmarkResult);
- my $msg;
- eval {
-
-
- my @SQLQuery = split /;/, $testHash{'sql'};
-
- # Throw out the last one if it is just space
- if ($SQLQuery[$#SQLQuery] =~ /^\s*$/) { $#SQLQuery--; }
-
- # If the last one is a comment, decrement the count
- if ($#SQLQuery > 0 && $SQLQuery[$#SQLQuery] !~ /select/i && $SQLQuery[$#SQLQuery] =~ /--/) {
- $#SQLQuery--;
- }
-
- $testHash{'queries'} = \@SQLQuery;
-
- $testResult = $self->runTest(\%testHash, $log, $copyResults);
- my $endTime = time;
-
- $benchmarkResult = $self->generateBenchmark(\%testHash, $log);
-
- my $result;
- if( $dontCompareThisOne ) {
- $result = 1;
- print $log "$0::$className::$subName TEST MARKED NOVERIFY <$testName>\n";
- } else {
- $result = $self->compare($testResult, $benchmarkResult, $log, \%testHash);
- print $log "Test $testName\n";
- }
-
- if ($result) {
- $msg = "TEST SUCCEEDED <$testName> at " . time . "\n";
- $testStatuses->{$testName} = $passedStr;
- } else {
- $msg= "TEST FAILED <$testName> at " . time . "\n";
- $testStatuses->{$testName} = $failedStr;
- }
- print $log $msg;
-
- $dbinfo{'duration'} = $endTime - $beginTime;
- $self->recordResults($result, $testResult, $benchmarkResult,
- \%dbinfo, $log);
- };
-
- if ($@) {
- my $endTime = time;
- print $log "$0::$subName FAILED: Failed to run test $testName <$@>\n";
- $testStatuses->{$testName} = $abortedStr;
- $dbinfo{'duration'} = $beginTime - $endTime;
- }
-
- eval {
- $dbinfo{'status'} = $testStatuses->{$testName};
- if($dbh) {
- $dbh->insertTestCase(\%dbinfo);
- }
- };
- if ($@) {
- chomp $@;
- warn "Failed to insert test case info, error <$@>\n";
- }
-
- $self->cleanup($testStatuses->{$testName}, \%testHash, $testResult,
- $benchmarkResult, $log);
+ # hope they don't have more than store per line
+ # also note that this won't work if you comment out a store
+ my @q = split(/\n/, $testCmd->{'pig'});
+ for (my $i = 0; $i < @q; $i++) {
+ $count += $q[$i] =~ /store\s+[a-zA-Z][a-zA-Z0-9_]*\s+into/i;
+ }
- #$report->testcase( $group->{'name'}, $testName, $duration, $msg, $testStatuses->{$testName}, $testResult ) if ( $report );
- $report->testcase( $group->{'name'}, $testName, $duration, $msg, $testStatuses->{$testName} ) if ( $report );
- $groupDuration = $groupDuration + $duration;
- $totalDuration = $totalDuration + $duration;
- printResults($testStatuses, $log, "Results so far");
- }
- if ( $report ) {
- my $reportname= $group->{'name'};
- if ( $globalHash{'reportname'} ) {
- $reportname= $globalHash{'reportname'};
- }
- $report->systemOut( $logname, $reportname );
- printGroupResultsXml( $report, $reportname, $testStatuses, $groupDuration );
- }
- $report = 0;
- $groupDuration=0;
+ return $count;
+}
- }
+##############################################################################
+# Check whether we should be running this test or not.
+#
+sub wrongExecutionMode($$)
+{
+ my ($self, $testCmd) = @_;
- # Do the global cleanup
- $self->globalCleanup(\%globalHash, $log);
+ # Check that we should run this test. If the current execution type
+ # doesn't match the execonly flag, then skip this one.
+ return (defined $testCmd->{'execonly'} &&
+ $testCmd->{'execonly'} ne $testCmd->{'exectype'});
}
##############################################################################
Added: pig/branches/branch-0.9/test/e2e/pig/macros/macro1.pig
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/e2e/pig/macros/macro1.pig?rev=1170769&view=auto
==============================================================================
--- pig/branches/branch-0.9/test/e2e/pig/macros/macro1.pig (added)
+++ pig/branches/branch-0.9/test/e2e/pig/macros/macro1.pig Wed Sep 14 19:03:12 2011
@@ -0,0 +1,4 @@
+define simple_macro(in_relation, min_gpa, max_age) returns c {
+ b = filter $in_relation by gpa >= $min_gpa and age <= $max_age;
+ $c = foreach b generate age, name;
+};
Added: pig/branches/branch-0.9/test/e2e/pig/macros/macro_bad1.pig
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/e2e/pig/macros/macro_bad1.pig?rev=1170769&view=auto
==============================================================================
--- pig/branches/branch-0.9/test/e2e/pig/macros/macro_bad1.pig (added)
+++ pig/branches/branch-0.9/test/e2e/pig/macros/macro_bad1.pig Wed Sep 14 19:03:12 2011
@@ -0,0 +1,6 @@
+-- macro with syntax error
+
+define simple_macro(in_relation, min_gpa, max_age) returns c {
+ b = fiter $in_relation by gpa >= $min_gpa and age <= $max_age;
+ c = foreach b generate age, name;
+};
Modified: pig/branches/branch-0.9/test/e2e/pig/tests/bigdata.conf
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/e2e/pig/tests/bigdata.conf?rev=1170769&r1=1170768&r2=1170769&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/e2e/pig/tests/bigdata.conf (original)
+++ pig/branches/branch-0.9/test/e2e/pig/tests/bigdata.conf Wed Sep 14 19:03:12 2011
@@ -24,7 +24,8 @@
$cfg = {
'driver' => 'Pig',
- 'nummachines' => 10,
+ 'execonly' => 'mapred',
+ 'execonly' => 'mapred',
'groups' => [
{
@@ -32,6 +33,8 @@ $cfg = {
'tests' => [
{
'num' => 1,
+ ,'floatpostprocess' => 1
+ ,'delimiter' => ' ',
'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
c = filter a by age < '50';
@@ -42,37 +45,21 @@ g = group f by registration parallel 20;
h = foreach g generate group, SUM(f.d::contributions);
i = order h by $1, $0 parallel 20;
store i into ':OUTPATH:';\,
- 'sql' => "select registration, sum(contributions)
-from studenttab20m as a full outer join votertab10k as b using (name, age)
-where a.age < 50 and b.age < 50
-group by registration
-order by 2, 1;",
- 'floatpostprocess' => 1,
- 'delimiter' => ' ',
- },
-# {
-# 'num' => 2,
-# 'sortBenchmark' => 1,
-# 'sortResults' => 1,
-# 'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
-#a1 = filter a by age < '50';
-#b = group a1 by (name, age);
-#c = foreach b generate group as g, AVG(a1.gpa);
-#d = filter c by $1 > '3.0';
-#d1 = foreach d generate g.$0 as name, g.$1 as age, $1 as gpa;
-#e = group d1 by name;
-#f = foreach e generate group, AVG(d1.age);
-#store f into ':OUTPATH:';\,
-# 'sql' => "select name, avg(age)
-#from (select name, age, avg(gpa) from studenttab20m where age < 50 group by name, age having avg(gpa) > 3.0) as T
-#group by name;",
-# 'floatpostprocess' => 1,
-# 'delimiter' => ' ',
-# },
+ },
+ {
+ 'num' => 2,
+ 'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
+a1 = filter a by age < '50';
+b = group a1 by (name, age);
+c = foreach b generate group as g, AVG(a1.gpa);
+d = filter c by $1 > 3.0;
+d1 = foreach d generate g.$0 as name, g.$1 as age, $1 as gpa;
+e = group d1 by name;
+f = foreach e generate group, AVG(d1.age);
+store f into ':OUTPATH:';\,
+ },
{
'num' => 3,
- 'sortBenchmark' => 1,
- 'sortResults' => 1,
'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
c = foreach a generate name;
@@ -81,70 +68,46 @@ e = filter c by name matches '.*allen$';
f = union d, e;
g = distinct f parallel 20;
store g into ':OUTPATH:';\,
- 'sql' => q\select name from studenttab20m where name like '%allen'
-union
-select name from votertab10k;\,
},
{
'num' => 4,
- 'sortBenchmark' => 1,
- 'sortResults' => 1,
'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
b = group a all parallel 20;
c = foreach b generate COUNT(a.$0);
store c into ':OUTPATH:';\,
- 'sql' => "select count(*) from studenttab20m;",
},
{
'num' => 5,
- 'sortBenchmark' => 1,
- 'sortResults' => 1,
'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
b = group a by name parallel 20;
c = foreach b generate group, COUNT($1);
store c into ':OUTPATH:';\,
- 'sql' => "select name, count(*) from studenttab20m group by name;",
},
{
'num' => 6,
- 'sortBenchmark' => 1,
- 'sortResults' => 1,
'pig' => q\
define cmd `perl PigStreaming.pl` ship(':SCRIPTHOMEPATH:/PigStreaming.pl') stderr('CMD' limit 3);
a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
b = stream a through cmd as (n, a, g);
c = foreach b generate n, a;
store c into ':OUTPATH:';\,
- 'sql' => "select name, age from studenttab20m;",
},
{
'num' => 7,
- 'sortBenchmark' => 1,
- 'sortResults' => 1,
'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
b = order a by name parallel 20;
store b into ':OUTPATH:';\,
- 'sql' => "select * from studenttab20m;",
'sortArgs' => ['-t', ' ', '+0', '-1'],
- 'floatpostprocess' => 1,
- 'delimiter' => ' ',
},
{
'num' => 8,
- 'sortBenchmark' => 1,
- 'sortResults' => 1,
'pig' => q\a = load ':INPATH:/singlefile/studenttab20m' as (name:chararray, age:int, gpa:double);
b = order a by name, age desc parallel 20;
store b into ':OUTPATH:';\,
- 'sql' => "select * from studenttab20m;",
'sortArgs' => ['-t', ' ', '+0', '-1', '+1rn', '-2'],
- 'floatpostprocess' => 1,
- 'delimiter' => ' ',
},
{
'num' => 9,
- 'sortBenchmark' => 1,
- 'sortResults' => 1,
'pig' => q\A = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
B = filter A by age > 20;
C = group B by name;
@@ -152,18 +115,8 @@ D = foreach C generate group, COUNT(B) P
E = order D by $0 PARALLEL 16;
F = limit E 10;
store F into ':OUTPATH:';\,
- 'sql' => "select name, count(*) from studenttab20m where age > 20 group by name order by name limit 10;",
'sortArgs' => ['-t', ' ', '+0', '-1'],
},
- {
- 'num' => 10,
- 'floatpostprocess' => 1,
- 'delimiter' => ' ',
- 'pig' =>q\a = load ':INPATH:/singlefile/studenttab20m' as (name, age, gpa);
-store a into ':OUTPATH:';\,
- 'sql' => "select name, age, gpa from studenttab20m;"
- }
-
]
},
Modified: pig/branches/branch-0.9/test/e2e/pig/tests/cmdline.conf
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/e2e/pig/tests/cmdline.conf?rev=1170769&r1=1170768&r2=1170769&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/e2e/pig/tests/cmdline.conf (original)
+++ pig/branches/branch-0.9/test/e2e/pig/tests/cmdline.conf Wed Sep 14 19:03:12 2011
@@ -35,6 +35,7 @@ $cfg = {
'name' => 'Describe_cmdline',
'floatpostprocess' => 0,
'delimiter' => ' ',
+ 'ignore' => 1, # Need to investigate failures
'tests' => [
#JIRA[PIG-372]
#JIRA[PIG-374]
@@ -148,7 +149,7 @@ describe D;\,
# C: {name: chararray,age: integer}
# D: {group: chararray C: {name: chararray,age: integer}}
'expected_out_regex' =>
-"A: {name: chararray,age: int,gpa: float}\nB: {name: chararray,age: int}\nC: {name: chararray,age: int}\nD: {group: chararray,C: {name: chararray,age: int}}\n",
+"A: {name: chararray,age: int,gpa: float}\nB: {name: chararray,age: int}\nC: {name: chararray,age: int}\nD: {group: chararray,C: {(name: chararray,age: int)}}",
},
{
@@ -168,7 +169,7 @@ describe D;\,
# C: {name: chararray,age: integer}
# D: {group: chararray C: {name: chararray,age: integer}}
'expected_out_regex' =>
-"A: {name: chararray,age: int,gpa: float}\nB: {name: chararray,age: int}\nC: {name: chararray,age: int}\nD: {group: chararray,C: {name: chararray,age: int}}\n",
+"A: {name: chararray,age: int,gpa: float}\nB: {name: chararray,age: int}\nC: {name: chararray,age: int}\nD: {group: chararray,C: {(name: chararray,age: int)}}\n",
},
{
@@ -188,7 +189,7 @@ describe A;\,
# C: {name: chararray,age: integer}
# D: {group: chararray C: {name: chararray,age: integer}}
'expected_out_regex' =>
-"D: {group: chararray,C: {name: chararray,age: int}}\nC: {name: chararray,age: int}\nB: {name: chararray,age: int}\nA: {name: chararray,age: int,gpa: float}",
+"D: {group: chararray,C: {(name: chararray,age: int)}}\nC: {name: chararray,age: int}\nB: {name: chararray,age: int}\nA: {name: chararray,age: int,gpa: float}",
},
{
@@ -247,6 +248,7 @@ dump A;\,
{
'name' => 'Warning',
'floatpostprocess' => 0,
+ 'execonly' => 'mapred', # Warnings use counters, which don't work in local mode
'delimiter' => ' ',
'tests' => [
@@ -297,6 +299,66 @@ describe b;\,
},
],
},
+ {
+
+# 1 Test that a nested foreach gives instant feedback: after user issue the foreach statement in Grunt
+# 2 Test that a nested foreach gives instant feedback: as part of execution: when we run this foreach statement, we will dump the schema for the nested alias
+# 3 Describe single Alias resulting from a nested Foreach
+# 4 Describe multiple Alias resulting from a nested Foreach
+# 5 Describe resulting from a nested Foreach that contains a positional parameter
+# 6 Describe for child Alias resulting from a nested Foreach where the child alias had multiple assignments The expected behavior is that the last assignment will destermine the result of the describe statement.
+# 7 Describe for an Alias resulting from a nested Foreach where the projection for the nested alias is empty
+# 8 Describe within a foreach statement
+# 9 Describe for alias with complex data types
+# 10 Describe that uses references an alias from an AS clause
+
+ 'name' => 'NestedDescribe',
+ 'tests' => [
+ {
+# 2 Test that a nested foreach gives instant feedback: as part of execution: when we run this foreach statement, we will dump the schema for the nested alias
+# 3 Describe single Alias resulting from a nested Foreach
+ 'num' => 1,
+ 'pig' => q\
+A = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
+B = group A by name;
+C = foreach B { D = distinct A.age; generate COUNT(D), group;}
+describe C::D;
+\,
+ ,'expected_out_regex' => "D: {age: bytearray}"
+
+ },{
+# 2 Test that a nested foreach gives instant feedback: as part of execution: when we run this foreach statement, we will dump the schema for the nested alias
+# 7 Describe for an Alias resulting from a nested Foreach where the projection for the nested alias is empty
+ 'num' => 2,
+ 'pig' => q\
+A = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
+B = group A by name;
+C = foreach B { D = distinct A.age; E= filter D by age > 1000; generate COUNT(E), group;}
+describe C;
+\,
+ ,'expected_out_regex' => "C: {long,group: bytearray}"
+
+
+ },{
+# 1 Test that a nested foreach gives instant feedback: after user issue the foreach statement in Grunt
+# 2 Test that a nested foreach gives instant feedback: as part of execution: when we run this foreach statement, we will dump the schema for the nested alias
+# 3 Describe single Alias resulting from a nested Foreach
+# 4 Describe multiple Alias resulting from a nested Foreach
+# 5 Describe resulting from a nested Foreach that contains a positional parameter
+ 'num' => 3,
+ 'pig' => q\
+A = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
+B = group A by name;
+C = foreach B { D = distinct A.$1; generate COUNT(D), group;}
+describe C::D;
+\,
+ ,'expected_out_regex' => "D: {age: bytearray}"
+
+
+ },
+ ]
+
+ }
]
}
;
Modified: pig/branches/branch-0.9/test/e2e/pig/tests/grunt.conf
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/e2e/pig/tests/grunt.conf?rev=1170769&r1=1170768&r2=1170769&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/e2e/pig/tests/grunt.conf (original)
+++ pig/branches/branch-0.9/test/e2e/pig/tests/grunt.conf Wed Sep 14 19:03:12 2011
@@ -36,448 +36,89 @@ $cfg = {
'name' => 'Grunt',
'tests' => [
{
-# TEST Simple Test using ls
- 'num' => 100,
- 'pig' => q\
-ls :INPATH:
-sh ls
-sh echo hello
-sh ls -l :PIGHARNESS:
-sh echo hello2
-\,
- ,'expected_out_regex' => "hdfs:.*singlefile"
- ,'expected_out_regex' => "hello"
- ,'expected_out_regex' => "hello2"
- ,'expected_out_regex' => "libexec"
- ,'rc' => 0
-
- },{
-# TEST Extensive Test using ls
'num' => 1,
- 'pig' => q\
-sh ls -a :PIGHARNESS:/in/sql/address.txt
-sh ls -l :PIGHARNESS:/in/sql/dates.txt
-sh ls :PIGHARNESS:
-sh bash -c 'ls :PIGHARNESS:/in/sql/student*'
-sh bash -c 'ls -l :PIGHARNESS:/in/sql/c*.txt'
-sh ls /bin
-ls
-\,
- ,'expected_out_regex' => "address.txt"
- ,'expected_out_regex' => "dates.txt"
- ,'expected_out_regex' => "libexec"
- ,'expected_out_regex' => "-r.-r.-r--"
- ,'expected_out_regex' => "student"
- ,'expected_out_regex' => "customer"
- ,'expected_out_regex' => "rmdir"
- ,'rc' => 0
-
+ 'pig' => "sh ls :PIGHARNESS:",
+ 'expected_out_regex' => "libexec",
+ 'rc' => 0
},{
-# TEST pwd
'num' => 2,
- 'pig' => q\
-pwd
-sh pwd
-\,
- ,'expected_out_regex' => "hdfs:"
- ,'rc' => 0
-
- },{
-# TEST for cat
- 'num' => 3,
- 'pig' => q\
-sh cat :PIGHARNESS:/in/sql/numbers.txt
-\,
- ,'expected_out_regex' => "5000:5000:2:906:5060:5060L:5000.2f:5000.2"
- ,'rc' => 0
+ 'pig' => "pwd",
+ 'execonly' => 'mapred', # don't have a clue what their cwd will be for local mode
+ 'expected_out_regex' => "hdfs:",
+ 'rc' => 0
},{
-# TEST for pipe using cat | wc
-#BROKEN should be fixed with nightly - pipe fixed
'num' => 4,
- 'pig' => q\
-sh bash -c 'cat :PIGHARNESS:/in/sql/numbers.txt | wc'
-\,
- ,'expected_out_regex' => "5000.*5000.*199455"
+ 'pig' => "ls :INPATH:",
+ ,'expected_out_regex' => ".*singlefile"
,'rc' => 0
-# TEST for cd
- },{
- 'num' => 5,
- 'pig' => q\
-sh bash -c cd :LOCALINPATH:
-\,
- ,'rc' => 0
-
-# TEST for touch
},{
'num' => 6,
'pig' => q\
-sh mkdir -p :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_6.test
-sh touch :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_6.test/newfile.txt
-sh ls -l :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_6.test/newfile.txt
-\,
- ,'expected_out_regex' => "Grunt_6.test/newfile.txt"
- ,'rc' => 0
- },{
-# TEST for rm
- 'num' => 7,
- 'pig' => q\
-sh mkdir -p :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_7.test
-sh touch :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_7.test/newfile.txt
-sh rm -fv :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_7.test/newfile.txt
-\,
- ,'expected_out_regex' => "removed.*Grunt_7.test/newfile.txt"
- ,'rc' => 0
- },{
-# TEST for mkdir
- 'num' => 8,
- 'pig' => q\
-sh mkdir -p :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_8.test/newdir
-sh touch :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_8.test/newdir/newfile.txt
-sh ls -l :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_8.test/newdir/newfile.txt
-\,
- ,'expected_out_regex' => "-r.-r.*newfile.txt"
- ,'rc' => 0
- },{
-# TEST for mkdir
-# TEST for chmod 775 file
- 'num' => 9,
- 'pig' => q\
-sh mkdir -p :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_9.test
-sh touch :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_9.test/newfile.txt
-sh chmod -v 755 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_9.test/newfile.txt
-\,
- ,'expected_out_regex' => "changed to 0755"
- ,'rc' => 0
- },{
- 'num' => 10,
- 'pig' => q\
-sh mkdir -vp :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_10.test/newdir
-sh rm -Rfv :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_10.test/newdir
-\,
- ,'expected_out_regex' => "mkdir: created directory.*newdir"
- ,'expected_out_regex' => "removed directory.*newdir"
- ,'rc' => 0
- },{
-# TEST for diff at directory level
- 'num' => 11,
- 'pig' => q\
-A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
-B = limit A 5;
-C = limit A 10;
-store B into ':OUTPATH:/B.out';
-store C into ':OUTPATH:/C.out';
-copyToLocal :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_11.tmp
-sh bash -c 'diff :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_11.tmp/B.out/part-r-00000 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_11.tmp/C.out/part-r-00000'
-\,
- ,'expected_out_regex' => "1000.*9.*1.*904.*5018.*5018.*904.501.*904.501"
- ,'rc' => 0
- },{
-# TEST for diff two file where one is empty file
- 'num' => 12,
- 'pig' => q\
-A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
-B = limit A 10;
-Empty = filter A by id > 99999;
-store B into ':OUTPATH:/B.out';
-store Empty into ':OUTPATH:/Empty.out';
-copyToLocal :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_12.test
-sh bash -c 'diff :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_12.test/B.out/part-r-00000 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_12.test/Empty.out/part-m-00000'
-\,
- ,'expected_out_regex'=>"1000.*10.*2.*900.*5051.*5051.*10.2.*10.2"
- ,'rc' => 0
-
- },{
-# TEST form chmod +x file
- 'num' => 13,
- 'pig' => q\
-A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
-B = limit A 10;
-store B into ':OUTPATH:';
-copyToLocal :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_13.test
-sh bash -c 'chmod -v -x :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_13.test/part-r-00000'
-\,
- ,'expected_out_regex' => "changed.*to.*0666.*rw-rw-rw-"
- ,'rc' => 0
-
- },{
-# TEST form chmod +775 file
- 'num' => 14,
- 'pig' => q\
-A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
-B = limit A 10;
-store B into ':OUTPATH:';
-copyToLocal :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_14.test
-sh bash -c 'chmod -v 775 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_14.test/part-r-00000'
-\,
- ,'expected_out_regex' => "changed.*to.*0775.*rwxrwxr-x"
-
- ,'rc' => 0
-
- },{
-# TEST for sort
-# TEST command "tail"
-# TEST multple unix commands in a single line
-#BROKEN NEEED UPDATE SHOULD WORK ON NIGHTLY
- 'num' => 15,
- 'pig' => q\
-A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
-B = limit A 10;
-store B into ':OUTPATH:';
-copyToLocal :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_15.test
-sh bash -c 'sort :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_15.test/part-r-00000 > :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_15.test/out.sorted'
-sh bash -c 'tail -n 2 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_15.test/out.sorted'
-\,
- ,'expected_out_regex' => "1000.*9.*1.*904.*5018.*5018.*904.501.*904.501"
-
- ,'rc' => 0
-
- },{
-# TEST for sort
-# TEST command "head"
-#BROKEN NEEED UPDATE SHOULD WORK ON NIGHTLY
- 'num' => 16,
- 'pig' => q\
-A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
-B = limit A 10;
-store B into ':OUTPATH:';
-copyToLocal :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_16.test
-sh bash -c 'sort :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_16.test/part-r-00000 > :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_16.test/out.sorted'
-sh bash -c 'head -n 2 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_16.test/out.sorted'
-\,
- ,'expected_out_regex' => "1000.*1.*3.*90.*5084.*5084.*8.3.*8.3"
-
- ,'rc' => 0
-
-
- },{
- 'num' => 17,
- 'pig' => q\
-sh whoami
-\,
- ,'expected_out_regex' => "hadoopqa"
- ,'rc' => 0
-
- },{
-# TEST for comment
-# TEST for wc
-# TEST for wc with pipe
-# COMMENTS NOT SUPPORTED
-# sh #THIS IS A COMMENT
-#sh cat :INPATH:/types/numbers.txt | wc
- 'num' => 18,
- 'pig' => q\
-copyToLocal :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_18.tmp/numbers.txt
-copyToLocal :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_18.tmp/numbers2.txt
-sh wc :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_18.tmp/numbers.txt
-sh wc -l :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_18.tmp/numbers2.txt
-\,
- ,'expected_out_regex' => "5000.*5000.*199455.*numbers.txt"
- ,'expected_out_regex' => "5000.*numbers2.txt"
- ,'rc' => 0
-
- },{
-# TEST for grep
-# TEST foringle quotes
-#BROKEN NEEED UPDATE SHOULD WORK ON NIGHTLY
-#May want t rewrite this so that we use pig to generate results to compare against
- 'num' => 19,
- 'pig' => q#
-copyToLocal :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_19.tmp/numbers.txt
-sh grep 503 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_19.tmp/numbers.txt
-sh grep :506*: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_19.tmp/numbers.txt
-sh bash -c 'grep \':501:\' :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_19.tmp/numbers.txt'
-#,
- ,'expected_out_regex' => "1000:501:5:90:5019:5019L:8.5f:8.5"
- ,'expected_out_regex' => "1000:503:3:90:5064:5064L:0.506490f:0.506490"
- ,'expected_out_regex' => "5000:4972:5:90:5066:5066L:5.9f:5.9"
- ,'rc' => 0
-
- },{
-# TEST for uniq
- 'num' => 20,
- 'pig' => q\
-copyToLocal :INPATH:/singlefile/studenttab10k :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_20.tmp/studenttab10k
-sh uniq :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_20.tmp/studenttab10k
-\,
- ,'expected_out_regex' => "gabriella.*brown.*60.*1.71"
- ,'rc' => 0
-
-# TEST for find
-# TEST for wildcard
-#sh find :INPATH: -name studenttab10k
- },{
- 'num' => 21,
- 'pig' => q\
-sh mkdir :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_21.tmp/singlefile
-copyToLocal :INPATH:/singlefile/studenttab10k :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_21.tmp/singlefile/studenttab10k
-sh find :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_21.tmp -name student*
-\,
- ,'expected_out_regex' => "studenttab10k"
- ,'rc' => 0
-
- },{
-# TEST for df
- 'num' => 22,
- 'pig' => q\
-sh df
-\,
- ,'expected_out_regex' => "Filesystem.*Used.*Available"
- ,'rc' => 0
-
- },{
-# TEST for du
- 'num' => 23,
- 'pig' => q\
-sh du :PIGHARNESS:/in
-\,
- ,'expected_out_regex' => "sql"
- ,'rc' => 0
-
- },{
-# TEST for comment - not supported
-#sh #IGNORE THIS
-# TEST for echo
- 'num' => 24,
- 'pig' => q\
-
-sh echo This little piggy went to market
-sh echo This little piggy stayed home
-sh echo This little piggy had roast beef
-sh echo This little piggy had none
-sh echo This little piggy went wee wee wee, all the way home
-\,
- ,'expected_out_regex' => "market"
- ,'expected_out_regex' => "stayed"
- ,'expected_out_regex' => "roast"
- ,'expected_out_regex' => "none"
- ,'expected_out_regex' => "all the way home"
- ,'rc' => 0
-
- },{
-# TEST for pipe
-#BROKEN NEEED UPDATE SHOULD WORK ON NIGHTLY
- 'num' => 25,
- 'pig' => q\
-
-sh bash -c 'echo This little piggy went to market | wc'
-sh bash -c 'echo This little piggy stayed home | wc'
-sh bash -c 'echo This little piggy had roast beef | wc'
-sh bash -c 'echo This little piggy had none | wc'
-sh bash -c 'echo This little piggy went wee wee wee, all the way home | wc'
-\,
- ,'expected_out_regex' => "1.*6.*33"
- ,'expected_out_regex' => "1.*5.*30"
- ,'expected_out_regex' => "1.*5.*27"
-
- ,'rc' => 0
-
-
- },{
-# TEST for environmet variable
- 'num' => 26,
- 'pig' => q\
-sh bash -c 'echo $PIG_HARNESS_ROOT'
-sh bash -c 'ls $PIG_HARNESS_ROOT'
-\,
- ,'expected_out_regex' => "libexec"
- ,'rc' => 0
-
-
-
-# TEST read file you don't have read rights for
- },{
- 'num' => 27,
- 'pig' => q\
sh touch /bin/bad
\,
,'expected_err_regex' => "Permission denied"
,'rc' => 0
- },{
- 'num' => 28,
- 'pig' => q\
-sh ls -l doesnotexist1
-sh ls -l doesnotexist2/bad
-sh ls -l :OUTPATH:/doesnotexist3.txt
-\,
- ,'expected_err_regex' => "doesnotexist1.*No such file or directory"
- ,'expected_err_regex' => "doesnotexist2/bad.*No such file or directory"
- ,'expected_err_regex' => "doesnotexist3.txt.*No such file or directory"
-
- ,'rc' => 0
-
- },{
-# TEST run a shell script
-# BUG-3990418 May result in "../../../../..: Permission denied" Error if the top
-# level directory does not have read permissions, this will not be fixed
-
- 'num' => 29,
- 'pig' => "
-sh $ENV{PIG_HARNESS_ROOT}/bin/run_fb.sh -h
-",
- ,'expected_out_regex' => "Usage.*run_fb.sh"
- ,'rc' => 0
- },{
-# TEST run awk script
-#BROKEN - removing this awk script for now
-# 'num' => 30,
-# 'pig' => q\
-#sh mkdir :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_30.tmp/types
-#copyToLocal :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_30.tmp/numbers.txt
-#sh bash -c 'awk '{ print }' :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_30.tmp/numbers.txt'
-#\,
-# ,'expected_err_regex' => "5000:5000:2:906:5060:5060L:5000.2f:5000.2"
-#
-# ,'rc' => 0
-# },{
-# TEST run sed script
- 'num' => 31,
- 'pig' => q\
-sh mkdir :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_31.tmp/types
-copyToLocal :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_31.tmp/types/numbers.txt
-sh bash -c 'sed -e "1d" :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_31.tmp/types/numbers.txt'
-\,
- ,'expected_err_regex' => "5000:5000:2:906:5060:5060L:5000.2f:5000.2"
- ,'rc' => 0
-
- },{
-#TEST run perl script
- 'num' => 32,
- 'pig' => q\
-sh mkdir :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_32.tmp/types
-copyToLocal :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_32.tmp/types/numbers.txt
-sh bash -c 'perl -p -e "s/:/+/g;" :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_32.tmp/types/numbers.txt'
-\,
- },{
-#TEST run perl script
-#BROKEN Causes a 2998 Unhandled Exception, check to see if fixed in nightly
- 'num' => 33,
- 'pig' => q\
-sh bash -c ls -l //grid/2/dev/pigqa/in/types; ls -l /grid/2/dev/pigqa/in/glob; ls -l //grid/2/dev/pigqa/in/types; ls -l /grid/2/dev/pigqa/in/glob; ls
-\,
-#sh bash -c 'ls -l //grid/2/dev/pigqa/in/types; ls -l /grid/2/dev/pigqa/in/glob; ls -l //grid/2/dev/pigqa/in/types; ls -l /grid/2/dev/pigqa/in/glob; ls'
- ,'expected_err_regex' => "address.txt"
-
- ,'rc' => 0
-
-#IGNORE THIS TEST ISN'T READY
-# },{
-# 'num' => 33,
-# 'pig' => q\
-#sh echo "export FOO=MyEnvVariable" > :OUTPATH:/env.txt
-#sh source :OUTPATH:/env.txt
-#sh source :OUTPATH:/env.txt; echo $FOO
-#\,
-# ,'expected_out_regex' => "E: {age: bytearray}"
-# ,'rc' => 0
-
- }
-
-#ADD
-#wildcards
-#Add sort with tab delimeter
+ },{
+ 'num' => 7,
+ 'pig' => "fs -ls :INPATH:",
+ 'expected_out_regex' => ".*singlefile",
+ 'rc' => 0
+ },{
+ 'num' => 8,
+ 'pig' => "cat :INPATH:/singlefile/studenttab10k",
+ 'expected_out_regex' => "gabriella brown\\t60\\t1.71",
+ 'rc' => 0
+ },{
+ 'num' => 9,
+ 'pig' => "cd :INPATH:
+ pwd",
+ 'expected_out_regex' => ".*data",
+ 'rc' => 0
+ },{
+ 'num' => 10,
+ 'pig' => "cp :INPATH:/singlefile/studenttab10k .
+ ls .",
+ 'expected_out_regex' => ".*studenttab10k",
+ 'rc' => 0
+ },{
+ 'num' => 11,
+ 'pig' => "cp :INPATH:/singlefile/studenttab10k ./fred
+ ls .",
+ 'expected_out_regex' => ".*fred",
+ 'rc' => 0
+ },{
+ 'num' => 12,
+ 'pig' => "cp :INPATH:/singlefile/studenttab10k ./jim
+ mv ./jim ./bob
+ ls .",
+ 'expected_out_regex' => ".*bob",
+ 'rc' => 0
+ },{
+ 'num' => 13,
+ 'pig' => "copyToLocal :INPATH:/singlefile/studenttab10k :TMP:
+ sh cat :TMP:/studenttab10k",
+ 'expected_out_regex' => "gabriella brown\\t60\\t1.71",
+ 'rc' => 0
+ },{
+ 'num' => 14,
+ 'pig' => "copyToLocal :INPATH:/singlefile/votertab10k :TMP:
+ copyFromLocal :TMP:/votertab10k ./joe
+ cat ./joe",
+ 'expected_out_regex' => "jessica xylophone\\t41\\tgreen\\t638.13",
+ 'rc' => 0
+ },{
+ 'num' => 15,
+ 'pig' => "rm fred bob joe",
+ 'not_expected_out_regex' => "joe",
+ 'rc' => 0
+ },{
+ 'num' => 16,
+ 'pig' => "rmf jill",
+ 'rc' => 0
+ }
]
},
],