You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2011/11/08 17:23:51 UTC
svn commit: r1199327 - in /pig/trunk: CHANGES.txt
test/e2e/pig/drivers/TestDriverPig.pm test/e2e/pig/tests/nightly.conf
Author: gates
Date: Tue Nov 8 16:23:50 2011
New Revision: 1199327
URL: http://svn.apache.org/viewvc?rev=1199327&view=rev
Log:
PIG-2352: e2e test harness' use of environment variables causes unintended effects between tests
Modified:
pig/trunk/CHANGES.txt
pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm
pig/trunk/test/e2e/pig/tests/nightly.conf
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1199327&r1=1199326&r2=1199327&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Nov 8 16:23:50 2011
@@ -34,6 +34,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-2352: e2e test harness' use of environment variables causes unintended effects between tests (gates)
+
Release 0.10.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm?rev=1199327&r1=1199326&r2=1199327&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm (original)
+++ pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm Tue Nov 8 16:23:50 2011
@@ -84,6 +84,7 @@ sub replaceParameters
# $cmd =~ s/:LOCALTESTPATH:/$testCmd->{'localpathbase'}/g;
$cmd =~ s/:BMPATH:/$testCmd->{'benchmarkPath'}/g;
$cmd =~ s/:TMP:/$testCmd->{'tmpPath'}/g;
+ $cmd =~ s/:HDFSTMP:/tmp\/$testCmd->{'runid'}/g;
if ( $testCmd->{'hadoopSecurity'} eq "secure" ) {
$cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteSecureCluster'}/g;
@@ -137,6 +138,11 @@ sub globalSetup
IPC::Run::run(['mkdir', '-p', $globalHash->{'tmpPath'}], \undef, $log, $log) or
die "Cannot create temporary directory " . $globalHash->{'tmpPath'} .
" " . "$ERRNO\n";
+
+ # Create the HDFS temporary directory
+ @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'mkdir', "tmp/$globalHash->{'runid'}");
+ print $log "Going to run " . join(" ", @cmd) . "\n";
+ IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n";
}
sub globalCleanup
@@ -146,6 +152,11 @@ sub globalCleanup
IPC::Run::run(['rm', '-rf', $globalHash->{'tmpPath'}], \undef, $log, $log) or
warn "Cannot remove temporary directory " . $globalHash->{'tmpPath'} .
" " . "$ERRNO\n";
+
+ # Cleanup the HDFS temporary directory
+ my @cmd = ($self->getPigCmd($globalHash, $log), '-e', 'fs', '-rmr', "tmp/$globalHash->{'runid'}");
+ print $log "Going to run " . join(" ", @cmd) . "\n";
+ IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n";
}
@@ -329,11 +340,12 @@ sub getPigCmd($$$)
if (defined($testCmd->{'java_params'})) {
$ENV{'PIG_OPTS'} = join(" ", @{$testCmd->{'java_params'}});
+ } else {
+ $ENV{'PIG_OPTS'} = undef;
}
-
print $log "Returning Pig command " . join(" ", @pigCmd) . "\n";
- print $log "With PIG_CLASSPATH set to " . $ENV{'PIG_CLASSPATH'} . "\n";
+ print $log "With PIG_CLASSPATH set to " . $ENV{'PIG_CLASSPATH'} . " and PIG_OPTS set to " . $ENV{'PIG_OPTS'} . "\n";
return @pigCmd;
}
Modified: pig/trunk/test/e2e/pig/tests/nightly.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/nightly.conf?rev=1199327&r1=1199326&r2=1199327&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/nightly.conf (original)
+++ pig/trunk/test/e2e/pig/tests/nightly.conf Tue Nov 8 16:23:50 2011
@@ -3889,14 +3889,13 @@ store E into ':OUTPATH:';\,
{
'num' => 1,
'pig' => "define bb BuildBloom('Hash.JENKINS_HASH', 'fixed', '128', '3');
- fs -rmr :TMP:/mybloom_1;
A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int, gpa:double);
B = filter A by name == 'alice allen';
C = group B all;
D = foreach C generate bb(B.name);
- store D into ':TMP:/mybloom_1';
+ store D into ':HDFSTMP:/mybloom_1';
exec;
- define bloom Bloom(':TMP:/mybloom_1');
+ define bloom Bloom(':HDFSTMP:/mybloom_1');
E = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int, gpa:double);
F = filter E by bloom(name);
store F into ':OUTPATH:';",
@@ -3908,14 +3907,13 @@ store E into ':OUTPATH:';\,
}, {
'num' => 2,
'pig' => "define bb BuildBloom('Hash.MURMUR_HASH', 'fixed', '128', '3');
- fs -rmr :TMP:/mybloom_2;
A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int, gpa:double);
B = filter A by name == 'alice allen';
C = group B all;
D = foreach C generate bb(B.name);
- store D into ':TMP:/mybloom_2';
+ store D into ':HDFSTMP:/mybloom_2';
exec;
- define bloom Bloom(':TMP:/mybloom_2');
+ define bloom Bloom(':HDFSTMP:/mybloom_2');
E = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int, gpa:double);
F = filter E by bloom(name);
G = load ':INPATH:/singlefile/votertab10k'as (name:chararray, age:int, reg:chararray, contrib:float);
@@ -3931,14 +3929,13 @@ store E into ':OUTPATH:';\,
},{
'num' => 3,
'pig' => "define bb BuildBloom('Hash.JENKINS_HASH', '1', '0.0001');
- fs -rmr :TMP:/mybloom_3;
A = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int, gpa:double);
B = filter A by name == 'alice allen';
C = group B all;
D = foreach C generate bb(B.name);
- store D into ':TMP:/mybloom_3';
+ store D into ':HDFSTMP:/mybloom_3';
exec;
- define bloom Bloom(':TMP:/mybloom_3');
+ define bloom Bloom(':HDFSTMP:/mybloom_3');
E = LOAD ':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int, gpa:double);
F = filter E by bloom(name);
G = load ':INPATH:/singlefile/votertab10k'as (name:chararray, age:int, reg:chararray, contrib:float);