You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/03/20 20:13:50 UTC
svn commit: r1579715 - in /pig/branches/tez/test/e2e/pig: build.xml
conf/tez.conf drivers/TestDriverPig.pm tests/tez.conf
Author: daijy
Date: Thu Mar 20 19:13:49 2014
New Revision: 1579715
URL: http://svn.apache.org/r1579715
Log:
PIG-3790: Several changes in Tez e2e
Removed:
pig/branches/tez/test/e2e/pig/tests/tez.conf
Modified:
pig/branches/tez/test/e2e/pig/build.xml
pig/branches/tez/test/e2e/pig/conf/tez.conf
pig/branches/tez/test/e2e/pig/drivers/TestDriverPig.pm
Modified: pig/branches/tez/test/e2e/pig/build.xml
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/e2e/pig/build.xml?rev=1579715&r1=1579714&r2=1579715&view=diff
==============================================================================
--- pig/branches/tez/test/e2e/pig/build.xml (original)
+++ pig/branches/tez/test/e2e/pig/build.xml Thu Mar 20 19:13:49 2014
@@ -59,6 +59,12 @@
<property name="benchmark.location" value="${test.location}/benchmarks"/>
+ <condition property="harness.old.pig" value="">
+ <not>
+ <isset property="harness.old.pig"/>
+ </not>
+ </condition>
+
<condition property="HADOOP_PREFIX" value="">
<not>
<isset property="HADOOP_PREFIX"/>
@@ -325,50 +331,10 @@
</antcall>
</target>
- <target name="test-tez" depends="property-check, udfs, tar, init-test">
- <!-- If they have not specified tests to run then null it out -->
- <property name="tests.to.run" value=""/>
- <!-- fork (parallelization) factors for e2e tests execution.
- Defaults are 1, which means *no* parellelization: -->
- <property name="fork.factor.group" value="1"/>
- <property name="fork.factor.conf.file" value="1"/>
- <property name="e2e.debug" value="false"/>
- <property name="harness.use.python" value="false"/>
-
- <exec executable="perl" dir="${test.location}" failonerror="true">
- <env key="HARNESS_ROOT" value="."/>
- <env key="HADOOP_VERSION" value="${hadoopversion}"/>
- <env key="PH_LOCAL" value="${harness.PH_LOCAL}"/>
- <env key="PH_OUT" value="${harness.PH_OUT}"/>
- <env key="PH_ROOT" value="."/>
- <env key="PH_PIG" value="${pig.dir}"/>
- <env key="HADOOP_CONF_DIR" value="${harness.cluster.conf}"/>
- <env key="PIG_USE_PYTHON" value="${harness.use.python}"/>
- <env key="PH_CLUSTER_BIN" value="${harness.cluster.bin}"/>
- <env key="PH_JYTHON_JAR" value="${jython.jar}"/>
- <env key="PH_JRUBY_JAR" value="${jruby.jar}"/>
- <env key="JRUBY_OPTS" value="--1.9"/>
- <env key="HARNESS_CONF" value="${basedir}/conf/tez.conf"/>
- <env key="HADOOP_HOME" value="${harness.hadoop.home}"/>
- <env key="HADOOP_PREFIX" value="${HADOOP_PREFIX}"/>
- <env key="HADOOP_COMMON_HOME" value="${HADOOP_COMMON_HOME}"/>
- <env key="HADOOP_HDFS_HOME" value="${HADOOP_HDFS_HOME}"/>
- <env key="HADOOP_MAPRED_HOME" value="${HADOOP_MAPRED_HOME}"/>
- <env key="YARN_CONF_DIR" value="${YARN_CONF_DIR}"/>
- <env key="YARN_HOME" value="${YARN_HOME}"/>
- <env key="OLD_HADOOP_HOME" value="${OLD_HADOOP_HOME}"/>
- <env key="PH_OLD_CLUSTER_CONF" value="${PH_OLD_CLUSTER_CONF}"/>
- <env key="PH_BENCHMARK_CACHE_PATH" value="${PH_BENCHMARK_CACHE_PATH}"/>
- <env key="HCAT_BIN" value="${hcat.bin}"/>
- <env key="PERL5LIB" value="${harness.PERL5LIB}"/>
- <env key="FORK_FACTOR_GROUP" value="${fork.factor.group}"/>
- <env key="FORK_FACTOR_FILE" value="${fork.factor.conf.file}"/>
- <env key="E2E_DEBUG" value="${e2e.debug}"/>
-
- <arg value="./test_harness.pl"/>
- <arg line="${tests.to.run}"/>
- <arg value="${test.location}/tests/tez.conf"/>
- </exec>
+ <target name="test-tez">
+ <antcall target="test-base">
+ <param name="harness.conf.file" value="${basedir}/conf/tez.conf"/>
+ </antcall>
</target>
<target name="deploy-base" depends="property-check, tar, init-test">
Modified: pig/branches/tez/test/e2e/pig/conf/tez.conf
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/e2e/pig/conf/tez.conf?rev=1579715&r1=1579714&r2=1579715&view=diff
==============================================================================
--- pig/branches/tez/test/e2e/pig/conf/tez.conf (original)
+++ pig/branches/tez/test/e2e/pig/conf/tez.conf Thu Mar 20 19:13:49 2014
@@ -53,6 +53,7 @@ $cfg = {
, 'paramPath' => "$ENV{PH_ROOT}/paramfiles"
, 'piggybankjarPath' => "$ENV{PH_PIG}/contrib/piggybank/java"
, 'pigpath' => "$ENV{PH_PIG}"
+ , 'oldpigpath' => "$ENV{PH_OLDPIG}"
, 'hcatbin' => "$ENV{HCAT_BIN}"
, 'usePython' => "$ENV{PIG_USE_PYTHON}"
, 'exectype' => 'tez'
Modified: pig/branches/tez/test/e2e/pig/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/e2e/pig/drivers/TestDriverPig.pm?rev=1579715&r1=1579714&r2=1579715&view=diff
==============================================================================
--- pig/branches/tez/test/e2e/pig/drivers/TestDriverPig.pm (original)
+++ pig/branches/tez/test/e2e/pig/drivers/TestDriverPig.pm Thu Mar 20 19:13:49 2014
@@ -128,6 +128,7 @@ sub globalSetup
$globalHash->{'outpath'} = $globalHash->{'outpathbase'} . "/" . $globalHash->{'runid'} . "/";
$globalHash->{'localpath'} = $globalHash->{'localpathbase'} . "/" . $globalHash->{'runid'} . "/";
$globalHash->{'tmpPath'} = $globalHash->{'tmpPath'} . "/" . $globalHash->{'runid'} . "/";
+ $globalHash->{'orig_pig_classpath'} = $ENV{'PIG_CLASSPATH'};
}
sub globalSetupConditional() {
@@ -399,7 +400,7 @@ sub getPigCmd($$$)
# Set it in our current environment. It will get inherited by the IPC::Run
# command.
- $ENV{'PIG_CLASSPATH'} = $ENV{'PIG_CLASSPATH'} . $separator . $pcp;
+ $ENV{'PIG_CLASSPATH'} = $testCmd->{'orig_pig_classpath'} . $separator . $pcp;
if ($testCmd->{'usePython'} eq "true") {
@pigCmd = ("python");
@@ -656,16 +657,12 @@ sub generateBenchmark
$modifiedTestCmd{'pig'} = $testCmd->{'pig_win'};
}
# Change so we're looking at the old version of Pig
- if (defined $testCmd->{'oldpigpath'}) {
+ if (defined $testCmd->{'oldpigpath'} && $testCmd->{'oldpigpath'} ne "") {
$modifiedTestCmd{'pigpath'} = $testCmd->{'oldpigpath'};
}
if (defined($testCmd->{'oldconfigpath'})) {
$modifiedTestCmd{'testconfigpath'} = $testCmd->{'oldconfigpath'};
}
- # For exectype tez, we compare tez with mapreduce
- if (defined $testCmd->{'benchmark_exectype'}) {
- $modifiedTestCmd{'exectype'} = $testCmd->{'benchmark_exectype'};
- }
# switch environment to old hadoop
$orighadoophome=$ENV{'HADOOP_HOME'};
$orighadoopconf=$ENV{'HADOOP_CONF_DIR'};
@@ -687,6 +684,10 @@ sub generateBenchmark
$ENV{'YARN_CONF_DIR'} = $ENV{'OLD_YARN_CONF_DIR'};
}
}
+ # For exectype tez, we compare tez with mapreduce
+ if (defined $testCmd->{'benchmark_exectype'}) {
+ $modifiedTestCmd{'exectype'} = $testCmd->{'benchmark_exectype'};
+ }
# Modify the test number so we don't run over the actual test output
# and logs
$modifiedTestCmd{'num'} = $testCmd->{'num'} . "_benchmark";