You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2011/09/13 03:21:55 UTC
svn commit: r1169999 - in /pig/trunk: CHANGES.txt test/e2e/pig/build.xml
test/e2e/pig/conf/default.conf test/e2e/pig/drivers/TestDriverPig.pm
test/e2e/pig/tests/nightly.conf
Author: daijy
Date: Tue Sep 13 01:21:55 2011
New Revision: 1169999
URL: http://svn.apache.org/viewvc?rev=1169999&view=rev
Log:
PIG-2272: e2e test harness should be able to set HADOOP_HOME
Modified:
pig/trunk/CHANGES.txt
pig/trunk/test/e2e/pig/build.xml
pig/trunk/test/e2e/pig/conf/default.conf
pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm
pig/trunk/test/e2e/pig/tests/nightly.conf
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1169999&r1=1169998&r2=1169999&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Sep 13 01:21:55 2011
@@ -239,6 +239,8 @@ Release 0.9.1 - Unreleased
IMPROVEMENTS
+PIG-2272: e2e test harness should be able to set HADOOP_HOME (gates via daijy)
+
PIG-2239: Pig should use "bin/hadoop jar pig-withouthadoop.jar" in bin/pig instead of forming java command itself (daijy)
PIG-2213: Pig 0.9.1 Documentation (chandec via daijy)
Modified: pig/trunk/test/e2e/pig/build.xml
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/build.xml?rev=1169999&r1=1169998&r2=1169999&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/build.xml (original)
+++ pig/trunk/test/e2e/pig/build.xml Tue Sep 13 01:21:55 2011
@@ -130,10 +130,26 @@
<target name="property-check">
<fail message="Please set the property harness.old.pig to the directory where your old version of Pig is installed"
unless="harness.old.pig"/>
- <fail message="Please set the property harness.cluster.conf to the conf directory of your hadoop installation"
- unless="harness.cluster.conf"/>
- <fail message="Please set the property harness.cluster.bin to the binary executable of your hadoop installation"
- unless="harness.cluster.bin"/>
+ <fail message="Please set the property harness.cluster.conf to the conf directory of your hadoop installation or harness.hadoop.home to HADOOP_HOME for your Hadoop installation.">
+ <condition>
+ <not>
+ <or>
+ <isset property="harness.cluster.conf"/>
+ <isset property="harness.hadoop.home"/>
+ </or>
+ </not>
+ </condition>
+ </fail>
+ <fail message="Please set the property harness.cluster.bin to the binary executable of your hadoop installation or harness.hadoop.home to HADOOP_HOME for your Hadoop installation.">
+ <condition>
+ <not>
+ <or>
+ <isset property="harness.cluster.bin"/>
+ <isset property="harness.hadoop.home"/>
+ </or>
+ </not>
+ </condition>
+ </fail>
</target>
<!-- Prep the test area -->
@@ -170,6 +186,7 @@
<env key="PH_CLUSTER_BIN" value="${harness.cluster.bin}"/>
<env key="PH_JYTHON_JAR" value="${jython.jar}"/>
<env key="HARNESS_CONF" value="${harness.conf.file}"/>
+ <env key="HADOOP_HOME" value="${harness.hadoop.home}"/>
<arg line="${tests.to.run}"/>
<arg value="${test.location}/tests/cmdline.conf"/>
<arg value="${test.location}/tests/multiquery.conf"/>
Modified: pig/trunk/test/e2e/pig/conf/default.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/conf/default.conf?rev=1169999&r1=1169998&r2=1169999&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/conf/default.conf (original)
+++ pig/trunk/test/e2e/pig/conf/default.conf Tue Sep 13 01:21:55 2011
@@ -52,7 +52,7 @@ $cfg = {
, 'exectype' => 'mapred'
#HADOOP
- , 'hadoopHome' => "$ENV{PH_ROOT}/lib"
+ , 'mapredjars' => "$ENV{PH_ROOT}/lib"
, 'userhomePath' => "$ENV{HOME}"
Modified: pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm?rev=1169999&r1=1169998&r2=1169999&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm (original)
+++ pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm Tue Sep 13 01:21:55 2011
@@ -74,7 +74,7 @@ sub replaceParameters
$cmd =~ s/:FUNCPATH:/$testCmd->{'funcjarPath'}/g;
$cmd =~ s/:RUNID:/$testCmd->{'UID'}/g;
$cmd =~ s/:USRHOMEPATH:/$testCmd->{'userhomePath'}/g;
- $cmd =~ s/:HADOOPHOME:/$testCmd->{'hadoopHome'}/g;
+ $cmd =~ s/:MAPREDJARS:/$testCmd->{'mapredjars'}/g;
$cmd =~ s/:SCRIPTHOMEPATH:/$testCmd->{'scriptPath'}/g;
$cmd =~ s/:DBUSER:/$testCmd->{'dbuser'}/g;
$cmd =~ s/:DBNAME:/$testCmd->{'dbdb'}/g;
@@ -308,7 +308,8 @@ sub getPigCmd($$$)
# set the PIG_CLASSPATH environment variable
my $pcp .= $testCmd->{'jythonjar'} if (defined($testCmd->{'jythonjar'}));
$pcp .= ":" . $testCmd->{'classpath'} if (defined($testCmd->{'classpath'}));
- $pcp .= ":" . $testCmd->{'testconfigpath'} if ($testCmd->{'exectype'} ne "local");
+ # Only add testconfigpath to PIG_CLASSPATH if HADOOP_HOME isn't defined
+ $pcp .= ":" . $testCmd->{'testconfigpath'} if ($testCmd->{'exectype'} ne "local"); #&& (! defined $ENV{'HADOOP_HOME'});
# Set it in our current environment. It will get inherited by the IPC::Run
# command.
Modified: pig/trunk/test/e2e/pig/tests/nightly.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/nightly.conf?rev=1169999&r1=1169998&r2=1169999&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/nightly.conf (original)
+++ pig/trunk/test/e2e/pig/tests/nightly.conf Tue Sep 13 01:21:55 2011
@@ -2938,7 +2938,7 @@ store e into ':OUTPATH:';\,
rmf table_testNativeMRJobSimple_input
rmf table_testNativeMRJobSimple_output
a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
-b = mapreduce ':HADOOPHOME:/hadoop-examples.jar' Store a into 'table_testNativeMRJobSimple_input' Load 'table_testNativeMRJobSimple_output' `wordcount table_testNativeMRJobSimple_input table_testNativeMRJobSimple_output`;
+b = mapreduce ':MAPREDJARS:/hadoop-examples.jar' Store a into 'table_testNativeMRJobSimple_input' Load 'table_testNativeMRJobSimple_output' `wordcount table_testNativeMRJobSimple_input table_testNativeMRJobSimple_output`;
store b into ':OUTPATH:';\,
'notmq' => 1,
},
@@ -2951,7 +2951,7 @@ rmf table_testNativeMRJobSimple_output
a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
b = foreach a generate name;
c = distinct b;
-d = mapreduce ':HADOOPHOME:/hadoop-examples.jar' Store c into 'table_testNativeMRJobSimple_input' Load 'table_testNativeMRJobSimple_output' as (name:chararray, count: int) `wordcount table_testNativeMRJobSimple_input table_testNativeMRJobSimple_output`;
+d = mapreduce ':MAPREDJARS:/hadoop-examples.jar' Store c into 'table_testNativeMRJobSimple_input' Load 'table_testNativeMRJobSimple_output' as (name:chararray, count: int) `wordcount table_testNativeMRJobSimple_input table_testNativeMRJobSimple_output`;
e = order d by name;
store e into ':OUTPATH:';\,
'sortArgs' => ['-t', ' '],
@@ -2964,7 +2964,7 @@ store e into ':OUTPATH:';\,
rmf table_testNativeMRJobSimple_input
rmf table_testNativeMRJobSimple_output
a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name, age, gpa);
-b = mapreduce ':HADOOPHOME:/hadoop-streaming.jar' Store a into 'table_testNativeMRJobSimple_input' Load 'table_testNativeMRJobSimple_output' as (name:chararray, count: int) `-input table_testNativeMRJobSimple_input -output table_testNativeMRJobSimple_output -mapper /bin/cat -reducer /usr/bin/wc`;
+b = mapreduce ':MAPREDJARS:/hadoop-streaming.jar' Store a into 'table_testNativeMRJobSimple_input' Load 'table_testNativeMRJobSimple_output' as (name:chararray, count: int) `-input table_testNativeMRJobSimple_input -output table_testNativeMRJobSimple_output -mapper /bin/cat -reducer /usr/bin/wc`;
store b into ':OUTPATH:';\,
'notmq' => 1,
},