You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2011/11/08 17:37:58 UTC
svn commit: r1199332 - in /pig/branches/branch-0.10: CHANGES.txt
test/e2e/pig/drivers/TestDriverPig.pm test/e2e/pig/tests/nightly.conf
Author: gates
Date: Tue Nov 8 16:37:58 2011
New Revision: 1199332
URL: http://svn.apache.org/viewvc?rev=1199332&view=rev
Log:
PIG-2352: e2e test harness' use of environment variables causes unintended effects between tests
Modified:
pig/branches/branch-0.10/CHANGES.txt
pig/branches/branch-0.10/test/e2e/pig/drivers/TestDriverPig.pm
pig/branches/branch-0.10/test/e2e/pig/tests/nightly.conf
Modified: pig/branches/branch-0.10/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/CHANGES.txt?rev=1199332&r1=1199331&r2=1199332&view=diff
==============================================================================
--- pig/branches/branch-0.10/CHANGES.txt (original)
+++ pig/branches/branch-0.10/CHANGES.txt Tue Nov 8 16:37:58 2011
@@ -148,6 +148,8 @@ PIG-2228: support partial aggregation in
BUG FIXES
+PIG-2352: e2e test harness' use of environment variables causes unintended effects between tests (gates)
+
PIG-2165: Need a way to deal with params and param_file in embedded pig in python (daijy)
PIG-2313: NPE in ILLUSTRATE trying to get StatusReporter in STORE (daijy)
Modified: pig/branches/branch-0.10/test/e2e/pig/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/test/e2e/pig/drivers/TestDriverPig.pm?rev=1199332&r1=1199331&r2=1199332&view=diff
==============================================================================
--- pig/branches/branch-0.10/test/e2e/pig/drivers/TestDriverPig.pm (original)
+++ pig/branches/branch-0.10/test/e2e/pig/drivers/TestDriverPig.pm Tue Nov 8 16:37:58 2011
@@ -84,6 +84,7 @@ sub replaceParameters
# $cmd =~ s/:LOCALTESTPATH:/$testCmd->{'localpathbase'}/g;
$cmd =~ s/:BMPATH:/$testCmd->{'benchmarkPath'}/g;
$cmd =~ s/:TMP:/$testCmd->{'tmpPath'}/g;
+ $cmd =~ s/:HDFSTMP:/tmp\/$testCmd->{'runid'}/g;
if ( $testCmd->{'hadoopSecurity'} eq "secure" ) {
$cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteSecureCluster'}/g;
@@ -137,6 +138,11 @@ sub globalSetup
IPC::Run::run(['mkdir', '-p', $globalHash->{'tmpPath'}], \undef, $log, $log) or
die "Cannot create temporary directory " . $globalHash->{'tmpPath'} .
" " . "$ERRNO\n";
+
+ # Create the HDFS temporary directory
+ @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'mkdir', "tmp/$globalHash->{'runid'}");
+ print $log "Going to run " . join(" ", @cmd) . "\n";
+ IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n";
}
sub globalCleanup
@@ -146,6 +152,11 @@ sub globalCleanup
IPC::Run::run(['rm', '-rf', $globalHash->{'tmpPath'}], \undef, $log, $log) or
warn "Cannot remove temporary directory " . $globalHash->{'tmpPath'} .
" " . "$ERRNO\n";
+
+ # Cleanup the HDFS temporary directory
+ my @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'fs', '-rmr', "tmp/$globalHash->{'runid'}");
+ print $log "Going to run " . join(" ", @cmd) . "\n";
+ IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n";
}
@@ -329,11 +340,12 @@ sub getPigCmd($$$)
if (defined($testCmd->{'java_params'})) {
$ENV{'PIG_OPTS'} = join(" ", @{$testCmd->{'java_params'}});
+ } else {
+ $ENV{'PIG_OPTS'} = undef;
}
-
print $log "Returning Pig command " . join(" ", @pigCmd) . "\n";
- print $log "With PIG_CLASSPATH set to " . $ENV{'PIG_CLASSPATH'} . "\n";
+ print $log "With PIG_CLASSPATH set to " . $ENV{'PIG_CLASSPATH'} . " and PIG_OPTS set to " . $ENV{'PIG_OPTS'} . "\n";
return @pigCmd;
}
Modified: pig/branches/branch-0.10/test/e2e/pig/tests/nightly.conf
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/test/e2e/pig/tests/nightly.conf?rev=1199332&r1=1199331&r2=1199332&view=diff
==============================================================================
--- pig/branches/branch-0.10/test/e2e/pig/tests/nightly.conf (original)
+++ pig/branches/branch-0.10/test/e2e/pig/tests/nightly.conf Tue Nov 8 16:37:58 2011
@@ -3889,14 +3889,13 @@ store E into ':OUTPATH:';\,
{
'num' => 1,
'pig' => "define bb BuildBloom('Hash.JENKINS_HASH', 'fixed', '128', '3');
- fs -rmr :TMP:/mybloom_1;
A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int, gpa:double);
B = filter A by name == 'alice allen';
C = group B all;
D = foreach C generate bb(B.name);
- store D into ':TMP:/mybloom_1';
+ store D into ':HDFSTMP:/mybloom_1';
exec;
- define bloom Bloom(':TMP:/mybloom_1');
+ define bloom Bloom(':HDFSTMP:/mybloom_1');
E = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int, gpa:double);
F = filter E by bloom(name);
store F into ':OUTPATH:';",
@@ -3908,14 +3907,13 @@ store E into ':OUTPATH:';\,
}, {
'num' => 2,
'pig' => "define bb BuildBloom('Hash.MURMUR_HASH', 'fixed', '128', '3');
- fs -rmr :TMP:/mybloom_2;
A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int, gpa:double);
B = filter A by name == 'alice allen';
C = group B all;
D = foreach C generate bb(B.name);
- store D into ':TMP:/mybloom_2';
+ store D into ':HDFSTMP:/mybloom_2';
exec;
- define bloom Bloom(':TMP:/mybloom_2');
+ define bloom Bloom(':HDFSTMP:/mybloom_2');
E = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int, gpa:double);
F = filter E by bloom(name);
G = load ':INPATH:/singlefile/votertab10k'as (name:chararray, age:int, reg:chararray, contrib:float);
@@ -3931,14 +3929,13 @@ store E into ':OUTPATH:';\,
},{
'num' => 3,
'pig' => "define bb BuildBloom('Hash.JENKINS_HASH', '1', '0.0001');
- fs -rmr :TMP:/mybloom_3;
A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int, gpa:double);
B = filter A by name == 'alice allen';
C = group B all;
D = foreach C generate bb(B.name);
- store D into ':TMP:/mybloom_3';
+ store D into ':HDFSTMP:/mybloom_3';
exec;
- define bloom Bloom(':TMP:/mybloom_3');
+ define bloom Bloom(':HDFSTMP:/mybloom_3');
E = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int, gpa:double);
F = filter E by bloom(name);
G = load ':INPATH:/singlefile/votertab10k'as (name:chararray, age:int, reg:chararray, contrib:float);