You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ma...@apache.org on 2014/09/22 10:02:45 UTC
svn commit: r1626686 - in /pig/branches/spark: build.xml
test/e2e/pig/build.xml test/e2e/pig/conf/spark.conf
test/e2e/pig/drivers/TestDriverPig.pm
Author: mayur
Date: Mon Sep 22 08:02:44 2014
New Revision: 1626686
URL: http://svn.apache.org/r1626686
Log:
PIG-4174: e2e tests for Spark setup
Added:
pig/branches/spark/test/e2e/pig/conf/spark.conf (with props)
Modified:
pig/branches/spark/build.xml
pig/branches/spark/test/e2e/pig/build.xml
pig/branches/spark/test/e2e/pig/drivers/TestDriverPig.pm
Modified: pig/branches/spark/build.xml
URL: http://svn.apache.org/viewvc/pig/branches/spark/build.xml?rev=1626686&r1=1626685&r2=1626686&view=diff
==============================================================================
--- pig/branches/spark/build.xml (original)
+++ pig/branches/spark/build.xml Mon Sep 22 08:02:44 2014
@@ -1005,6 +1005,10 @@
<ant dir="${test.e2e.dir}" target="test-tez"/>
</target>
+ <target name="test-e2e-spark" depends="jar, piggybank" description="run end-to-end tests in tez mode">
+ <ant dir="${test.e2e.dir}" target="test-spark"/>
+ </target>
+
<target name="test-e2e-deploy" depends="jar" description="deploy end-to-end tests to existing cluster">
<ant dir="${test.e2e.dir}" target="deploy"/>
</target>
Modified: pig/branches/spark/test/e2e/pig/build.xml
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/e2e/pig/build.xml?rev=1626686&r1=1626685&r2=1626686&view=diff
==============================================================================
--- pig/branches/spark/test/e2e/pig/build.xml (original)
+++ pig/branches/spark/test/e2e/pig/build.xml Mon Sep 22 08:02:44 2014
@@ -335,6 +335,12 @@
</antcall>
</target>
+ <target name="test-spark">
+ <antcall target="test-base">
+ <param name="harness.conf.file" value="${basedir}/conf/spark.conf"/>
+ </antcall>
+ </target>
+
<target name="deploy-base" depends="property-check, tar, init-test">
<exec executable="perl" dir="${test.location}" failonerror="true">
<env key="HARNESS_ROOT" value="."/>
Added: pig/branches/spark/test/e2e/pig/conf/spark.conf
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/e2e/pig/conf/spark.conf?rev=1626686&view=auto
==============================================================================
--- pig/branches/spark/test/e2e/pig/conf/spark.conf (added)
+++ pig/branches/spark/test/e2e/pig/conf/spark.conf Mon Sep 22 08:02:44 2014
@@ -0,0 +1,75 @@
+############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+my $me = `whoami`;
+chomp $me;
+
+# The contents of this file can be rewritten to fit your installation.
+# Also, you can define the following environment variables and set things up as in the test setup
+# PH_ROOT Root directory where test harness is installed
+# PH_LOCAL Root directory for input and output for local mode tests
+# PH_OUT Root directory where output data will be stored (on local disk, not HDFS)
+# PH_CLUSTER_BIN Conf directory for cluster being used
+# HADOOP_CONF_DIR Binary executable for cluster being used
+# PH_PIG Root directory for Pig version being used
+
+my $hdfsBase = $ENV{PH_HDFS_BASE} || "/user/pig";
+
+$cfg = {
+ #HDFS
+ 'inpathbase' => "$hdfsBase/tests/data"
+ , 'outpathbase' => "$hdfsBase/out"
+
+ #LOCAL
+ , 'localinpathbase' => "$ENV{PH_LOCAL}/in"
+ , 'localoutpathbase' => "$ENV{PH_LOCAL}/out/log"
+ , 'localxmlpathbase' => "$ENV{PH_LOCAL}/out/xml"
+ , 'localpathbase' => "$ENV{PH_LOCAL}/out/pigtest/$me"
+ , 'benchmarkcachepath'=> "$ENV{PH_BENCHMARK_CACHE_PATH}"
+
+ #TEST
+ , 'benchmarkPath' => "$ENV{PH_OUT}/benchmarks"
+ , 'scriptPath' => "$ENV{PH_ROOT}/libexec"
+ , 'tmpPath' => '/tmp/pigtest'
+
+ #PIG
+ , 'testconfigpath' => "$ENV{HADOOP_CONF_DIR}"
+ , 'funcjarPath' => "$ENV{PH_ROOT}/lib/java"
+ , 'paramPath' => "$ENV{PH_ROOT}/paramfiles"
+ , 'piggybankjarPath' => "$ENV{PH_PIG}/contrib/piggybank/java"
+ , 'pigpath' => "$ENV{PH_PIG}"
+ , 'oldpigpath' => "$ENV{PH_OLDPIG}"
+ , 'hcatbin' => "$ENV{HCAT_BIN}"
+ , 'usePython' => "$ENV{PIG_USE_PYTHON}"
+ , 'exectype' => 'spark'
+ , 'benchmark_exectype' => 'local'
+
+ #HADOOP
+ , 'mapredjars' => "$ENV{PH_ROOT}/lib"
+
+ #HIVE
+ , 'hivelibdir' => "$ENV{PH_HIVE_LIB_DIR}"
+ , 'hiveversion' => "$ENV{PH_HIVE_VERSION}"
+ , 'hiveshimsversion' => "$ENV{PH_HIVE_SHIMS_VERSION}"
+
+ , 'userhomePath' => "$ENV{HOME}"
+ ,'local.bin' => '/usr/bin'
+
+ ,'logDir' => "$ENV{PH_OUT}/log"
+ ,'propertiesFile' => "./conf/testpropertiesfile.conf"
+ ,'harness.console.level' => 'ERROR'
+
+};
Propchange: pig/branches/spark/test/e2e/pig/conf/spark.conf
------------------------------------------------------------------------------
svn:eol-style = native
Modified: pig/branches/spark/test/e2e/pig/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/pig/branches/spark/test/e2e/pig/drivers/TestDriverPig.pm?rev=1626686&r1=1626685&r2=1626686&view=diff
==============================================================================
--- pig/branches/spark/test/e2e/pig/drivers/TestDriverPig.pm (original)
+++ pig/branches/spark/test/e2e/pig/drivers/TestDriverPig.pm Mon Sep 22 08:02:44 2014
@@ -280,10 +280,13 @@ sub runPigCmdLine
if ($testCmd->{'exectype'} eq "tez") {
push(@baseCmd, ("-x", "tez"));
}
+ if ($testCmd->{'exectype'} eq "spark") {
+ push(@baseCmd, ("-x", "spark"));
+ }
my @cmd = @baseCmd;
# Add option -l giving location for secondary logs
- ##!!! Should that even be here?
+ ##!!! Should that even be here?
my $locallog = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".log";
push(@cmd, "-logfile");
push(@cmd, $locallog);
@@ -506,6 +509,9 @@ sub runPig
if ($testCmd->{'exectype'} eq "tez") {
push(@baseCmd, ("-x", "tez"));
}
+ if ($testCmd->{'exectype'} eq "spark") {
+ push(@baseCmd, ("-x", "spark"));
+ }
my @cmd = @baseCmd;
# Add option -l giving location for secondary logs