You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ha...@apache.org on 2011/08/19 23:15:51 UTC
svn commit: r1159790 - in /pig/trunk: ./ test/e2e/harness/
test/e2e/harness/test/conf/ test/e2e/pig/ test/e2e/pig/conf/
test/e2e/pig/deployers/ test/e2e/pig/drivers/
Author: hashutosh
Date: Fri Aug 19 21:15:46 2011
New Revision: 1159790
URL: http://svn.apache.org/viewvc?rev=1159790&view=rev
Log:
PIG-2196: Test harness should be independent of Pig
Added:
pig/trunk/test/e2e/harness/test_harness.pl
Removed:
pig/trunk/test/e2e/harness/pig_test_harness.pl
Modified:
pig/trunk/CHANGES.txt
pig/trunk/test/e2e/harness/Log.pm
pig/trunk/test/e2e/harness/Properties.pm
pig/trunk/test/e2e/harness/TestReport.pm
pig/trunk/test/e2e/harness/build.xml
pig/trunk/test/e2e/harness/test/conf/default.conf
pig/trunk/test/e2e/pig/build.xml
pig/trunk/test/e2e/pig/conf/default.conf
pig/trunk/test/e2e/pig/conf/existing_deployer.conf
pig/trunk/test/e2e/pig/conf/testpropertiesfile.conf
pig/trunk/test/e2e/pig/deployers/ExistingClusterDeployer.pm
pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1159790&r1=1159789&r2=1159790&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Aug 19 21:15:46 2011
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-2196: Test harness should be independent of Pig (hashutosh)
+
PIG-1429: Add Boolean Data Type to Pig (zjshen via daijy)
PIG-2218: Pig end-to-end tests should be accessible from top level build.xml (gates)
Modified: pig/trunk/test/e2e/harness/Log.pm
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/harness/Log.pm?rev=1159790&r1=1159789&r2=1159790&view=diff
==============================================================================
--- pig/trunk/test/e2e/harness/Log.pm (original)
+++ pig/trunk/test/e2e/harness/Log.pm Fri Aug 19 21:15:46 2011
@@ -90,9 +90,9 @@ sub new() {
my $usage= "USAGE: $class.new( Properties )"
. "\nMust set the following properties:"
- . "\tpigharness.log"
- . "\tpigharness.log.level"
- . "\tpigharness.console.level"
+ . "\tharness.log"
+ . "\tharness.log.level"
+ . "\tharness.console.level"
;
#################
@@ -106,7 +106,7 @@ sub new() {
#################
# Verify Log File
- my $logName = $self->{Properties}{'pigharness.log'};
+ my $logName = $self->{Properties}{'harness.log'};
die "FATAL ERROR $class.new() at ".__LINE__." : no log name specified in properties file\n$usage\n" if ( !$logName );
open( LOG, ">>$logName" )
@@ -118,7 +118,7 @@ sub new() {
#################
# Set Logging level
- my $logLevel= $self->{Properties}{'pigharness.log.level'};
+ my $logLevel= $self->{Properties}{'harness.log.level'};
if ( ! $logLevel ){
$logLevel = 'DEBUG';
} else {
@@ -128,7 +128,7 @@ sub new() {
# Set Console level
- my $consoleLevel = $self->{Properties}{'pigharness.console.level'};
+ my $consoleLevel = $self->{Properties}{'harness.console.level'};
if ( !$consoleLevel ){
$consoleLevel = 'ERROR';
Modified: pig/trunk/test/e2e/harness/Properties.pm
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/harness/Properties.pm?rev=1159790&r1=1159789&r2=1159790&view=diff
==============================================================================
--- pig/trunk/test/e2e/harness/Properties.pm (original)
+++ pig/trunk/test/e2e/harness/Properties.pm Fri Aug 19 21:15:46 2011
@@ -279,7 +279,7 @@ sub getPropertyValuesByFilter(){
#
#sub main() {
#
-#my $properties= new Properties( 1, "$ENV{'PIG_HARNESS_ROOT'}/conf/pig_test.conf" );
+#my $properties= new Properties( 1, "$ENV{'HARNESS_ROOT'}/conf/test.conf" );
#}
#&main();
Modified: pig/trunk/test/e2e/harness/TestReport.pm
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/harness/TestReport.pm?rev=1159790&r1=1159789&r2=1159790&view=diff
==============================================================================
--- pig/trunk/test/e2e/harness/TestReport.pm (original)
+++ pig/trunk/test/e2e/harness/TestReport.pm Fri Aug 19 21:15:46 2011
@@ -42,11 +42,11 @@ use IO::Handle;
#
my $ROOT=undef;
-if (defined $ENV{'PIG_HARNESS_ROOT'} ){
- $ROOT= $ENV{'PIG_HARNESS_ROOT'};
+if (defined $ENV{'HARNESS_ROOT'} ){
+ $ROOT= $ENV{'HARNESS_ROOT'};
} else {
- die "FATAL ERROR: $0 - You must set PIG_HARNESS_ROOT to the root directory of the pig_harness";
+ die "FATAL ERROR: $0 - You must set HARNESS_ROOT to the root directory of the harness";
}
unshift( @INC, "$ROOT/libexec" );
Modified: pig/trunk/test/e2e/harness/build.xml
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/harness/build.xml?rev=1159790&r1=1159789&r2=1159790&view=diff
==============================================================================
--- pig/trunk/test/e2e/harness/build.xml (original)
+++ pig/trunk/test/e2e/harness/build.xml Fri Aug 19 21:15:46 2011
@@ -18,8 +18,8 @@
<project name="TestHarness" default="tar">
<property name="tar.dir" value="${basedir}/tar"/>
- <property name="pig.testharness.dir" value="${basedir}"/>
- <property name="tar.name" value="pigharness.tar"/>
+ <property name="testharness.dir" value="${basedir}"/>
+ <property name="tar.name" value="harness.tar"/>
<property name="test.dist.dir" value="${basedir}/testdist"/>
<property name="test.dist.conf.dir" value="${test.dist.dir}/conf"/>
<property name="test.dir" value="${basedir}/test"/>
@@ -31,7 +31,7 @@
description="Create tar file with test harness modules and script">
<mkdir dir="${tar.dir}"/>
<copy todir="${tar.dir}">
- <fileset dir="${pig.testharness.dir}">
+ <fileset dir="${testharness.dir}">
<include name="Insert2Mysql.pm"/>
<include name="TestDeployerFactory.pm"/>
<include name="TestDeployer.pm"/>
@@ -40,7 +40,7 @@
<include name="TestReport.pm"/>
<include name="Log.pm"/>
<include name="Properties.pm"/>
- <include name="pig_test_harness.pl"/>
+ <include name="test_harness.pl"/>
</fileset>
</copy>
@@ -66,12 +66,12 @@
</copy>
<untar src="${tar.name}" dest="${test.dist.dir}"/>
- <chmod file="${test.dist.dir}/pig_test_harness.pl" perm="u+x"/>
+ <chmod file="${test.dist.dir}/test_harness.pl" perm="u+x"/>
</target>
<!-- Test the ability to run tests -->
<target name="test-test" depends="test-init">
- <exec executable="./pig_test_harness.pl" dir="${test.dist.dir}">
+ <exec executable="./test_harness.pl" dir="${test.dist.dir}">
<env key="PIG_HARNESS_ROOT" value="."/>
<arg value="test.conf"/>
</exec>
@@ -79,7 +79,7 @@
<!-- Test the deployer interface -->
<target name="test-deploy" depends="test-init">
- <exec executable="./pig_test_harness.pl" dir="${test.dist.dir}">
+ <exec executable="./test_harness.pl" dir="${test.dist.dir}">
<env key="PIG_HARNESS_ROOT" value="."/>
<arg value="-deploycfg"/>
<arg value="deploy.conf"/>
Modified: pig/trunk/test/e2e/harness/test/conf/default.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/harness/test/conf/default.conf?rev=1159790&r1=1159789&r2=1159790&view=diff
==============================================================================
--- pig/trunk/test/e2e/harness/test/conf/default.conf (original)
+++ pig/trunk/test/e2e/harness/test/conf/default.conf Fri Aug 19 21:15:46 2011
@@ -23,9 +23,9 @@ $cfg = {
, 'localxmlpathbase' => "./xml"
, 'localpathbase' => "."
- ,'pigharness.log' => './harness.log'
- ,'pigharness.log.level' => 'DEBUG'
- ,'pigharness.console.level'=> 'INFO'
+ ,'harness.log' => './harness.log'
+ ,'harness.log.level' => 'DEBUG'
+ ,'harness.console.level'=> 'INFO'
}
;
Added: pig/trunk/test/e2e/harness/test_harness.pl
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/harness/test_harness.pl?rev=1159790&view=auto
==============================================================================
--- pig/trunk/test/e2e/harness/test_harness.pl (added)
+++ pig/trunk/test/e2e/harness/test_harness.pl Fri Aug 19 21:15:46 2011
@@ -0,0 +1,538 @@
+#!/usr/bin/env perl
+
+############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+
+###########################################################################
+# Package: pig_test_harness
+#
+# This is the top level bootstraping script for the pig test harness,
+#
+#
+#
+# SYNOPSIS:
+#
+# pig_test_harness -help | -c <cluster> | -h <dir> | (-testjar <jar> -testconfigpath <path>) [-r <retention_days>] [-latest yes] [-x local] [-cleanuponly] [-secretDebugCmd] [-t <testcase>] <configfile>
+# pig_test_harness [OPTIONS] conffile [... confile]
+#
+# - OPTIONS:
+# -conf <harness config file> - set name of global harness config file
+# -l <log file name> - set log file name
+# -t <test group name> - set test group testcases
+# -d <description name> - set description for MySQL database
+# -r <regexp> - set regular expression for test group testcases
+# -db <0 or 1> - disable using MySQL database if set to 0
+# -st <group name> - start test from provided group name
+#
+#
+# Dependencies:
+#
+# The main program relies on three configuration/properties files
+# - pig_deploy.properties
+# - pigtest.properties
+# - pig_test_harness/test_harness.conf
+#
+# It will look for the properties file under: $ROOT/conf
+#
+#
+# Returns:
+#
+# 0 if no failures and no errors,
+# the sum of failures and errors otherwise
+
+
+
+# TODO
+# 1. Add -regex option.
+
+use strict;
+use File::Path;
+use Getopt::Long;
+use Cwd;
+
+
+
+# Var: $ROOT
+# The root directory for the pig harness.
+#
+# The main pogram relies on the top level directory $ROOT to be set
+# to the root directory of the harness, it sets it as follows:
+#
+
+
+our $ROOT = (defined($ENV{'PIG_HARNESS_ROOT'}) ? $ENV{'PIG_HARNESS_ROOT'} :
+ die "FATAL ERROR: $0 - You must set PIG_HARNESS_ROOT to the root directory of the pig_harness");
+
+unshift( @INC, "$ROOT/libexec" );
+unshift( @INC, ".");
+
+require TestDriver;
+require TestDriverFactory;
+require TestDeployer;
+require TestDeployerFactory;
+require Insert2Mysql;
+require Properties;
+require Log; # why aren't we using log4perl?
+
+# Var: $dblog
+# The database log. This is set in config file.
+our $dblog;
+
+##############################################################################
+# Sub: usage
+# Print usage statement
+#
+# Returns:
+# usage string
+sub usage
+{
+ return
+"Usage: $0 [OPTIONS] conffile [... confile]
+ OPTIONS:
+ -l <log file name> - set log file name
+ -t <test group name> - set test group testcases
+ -d <description name> - set description for MySQL database
+ -r <regexp> - set regular expression for test group testcases
+ -db <0 or 1> - disable using MySQL database if set to 0
+ -st <group name> - start test from provided group name
+ -deploycfg <deploy cfg file> -deploy - Deploy the test setup before testing
+ <deploy cfg file> is the configuration file for deployment
+ -deploycfg <deploy cfg file> -undeploy - Undeploy the test setup after testing
+ <deploy cfg file> is the configuration file for deployment
+ ";
+}
+
+##############################################################################
+# Sub: readCfg
+# Read the configuration file. The config file is in Perl format so we'll
+# just eval it. If anything goes wrong we'll complain and quit.
+#
+# Var: cfgFile
+# Full path name of config file
+#
+# Returns:
+# returns reference to hash built from cfg file.
+#
+
+sub readCfg($)
+{
+ my $cfgFile = shift;
+
+ open CFG, "< $cfgFile" or die "FATAL ERROR $0 at ".__LINE__.": Can't open $cfgFile, $!\n";
+
+ my $cfgContents;
+
+ $cfgContents .= $_ while (<CFG>);
+
+ close CFG;
+
+ my $cfg = undef;
+ eval("$cfgContents");
+ #my $cfg = eval("$cfgContents");
+
+ if ($@) {
+ chomp $@;
+ die "FATAL ERROR $0 at ".__LINE__." : Error reading config file <$cfgFile>, <$@>\n";
+ }
+
+ if (not defined $cfg) {
+ die "FATAL ERROR $0 at ".__LINE__." : Configuration file <$cfgFile> should have defined \$cfg\n";
+ }
+
+ # Add the name of the file
+ $cfg->{'file'} = $cfgFile;
+
+ return $cfg;
+}
+
+
+
+##############################################################################
+# Sub:
+#
+# Var: %testStatuses
+# A hash containing the resutls from each test : pass, fail, error
+#
+# Returns: int
+# returns 0 if no failures and no errors,
+# the sum of failures and errors otherwise
+#
+
+sub exitStatus
+{
+ my ($testStatuses) = @_;
+ my $subName = (caller(0))[3];
+
+ my $passedStr = 'passed';
+ my $failedStr = 'failed';
+ my $abortedStr = 'aborted';
+ my $dependStr = 'failed_dependency';
+ my $skippedStr = 'skipped';
+
+ my ($pass, $fail, $abort, $depend, $skipped) = (0, 0, 0, 0, 0);
+
+ foreach (keys(%$testStatuses)) {
+ ($testStatuses->{$_} eq $passedStr) && $pass++;
+ ($testStatuses->{$_} eq $failedStr) && $fail++;
+ ($testStatuses->{$_} eq $abortedStr) && $abort++;
+ ($testStatuses->{$_} eq $dependStr) && $depend++;
+ ($testStatuses->{$_} eq $skippedStr) && $skipped++;
+ }
+
+ return ($fail + $abort);
+}
+
+
+##############################################################################
+# Sub: main
+# Gets the corresponding test driver and runs the tests.
+#
+# - Reads the global config file.
+# - Reads ARGV as described in "usage"
+# - Attaches to the database and gets a test run id.
+# - Loads the configuration file.
+# - Gets the test driver that will be used to parse test file.
+# - Runs the tests by invoking the run command from the test driver.
+# - Prints the final results.
+#
+#
+# Var: $logfile
+# The test run log name.
+# If no logfile is specified , then it is configured as follows :
+# The file name for the test result log. The location of the log directory
+# is obtained from the pig configuration value . The filename
+# is stored as $globalCfg{localoutpathbase}/pig_test_harness_log_{time}
+#
+# Var: $testrun_desc
+# A description of the test run to be recorded in the logs.
+#
+# Var: @testgroups
+# A list of all the test groups. This is passed in as a command line option.
+#
+# Var: @testMatches
+# A list of test patterns specified by the "-t option" . If none was passed then all tests match.
+#
+# Var: $globalCfg
+# All values to be shared globally. The $harnessCfg values are stored in the globalConfig
+#
+# Var: $harnessCfg
+# The pig configuration file. It assumes it is located at
+# $ROOT/conf/pig_test_harness/test_harness.conf.
+#
+# Var: $log
+# The pig test log.
+#
+# Var: $dbh
+# Instance of Insert2Mysal, this object provides database access subroutines.
+#
+# Var: %testStatuses
+# A hash containing the resutls from each test : pass, fail, error
+
+
+my $logfile = "";
+my $testrun_desc = 'none';
+my @testgroups;
+my @testMatches;
+my $startat = undef;
+my $deploycfg = undef;
+my $deploy = undef;
+my $undeploy = undef;
+my $help=0;
+
+die usage() if (@ARGV == 0);
+
+# Arguments on the command line can override values in the conf file
+# so first read in the conf file, then process the arguments.
+# But conf file can be specified on command line, so
+# get that from ARGV before processing other arguments.
+
+# Find the harness config file
+# 1) Use command line option -conf if given
+# 2) else use env var PIG_HARNESS_CONF if set
+# 3) else use default.conf if found
+# 4) else fail
+
+my $harnessCfg = "";
+for (my $i = 0; $i < @ARGV; $i++) {
+ if ($ARGV[$i] eq "-conf") {
+ $harnessCfg = $ARGV[$i + 1];
+ splice(@ARGV, $i, 2);
+ last;
+ }
+}
+if ($harnessCfg eq "") {
+ if (defined($ENV{'PIG_HARNESS_CONF'})) {
+ $harnessCfg = $ENV{'PIG_HARNESS_CONF'};
+ } else {
+ $harnessCfg = "$ROOT/conf/default.conf";
+ }
+}
+
+
+# Read the global config file
+my $globalCfg = "";
+if ( -e "$harnessCfg" ) {
+ $globalCfg = readCfg("$harnessCfg");
+ $globalCfg->{'harnessCfg'} = $harnessCfg;
+
+} else {
+ die "FATAL ERROR: $0 at ".__LINE__." - Configuration file <$harnessCfg> does NOT exist\n";
+}
+
+# *pig* -help | -c <cluster> | -h <dir> | (-testjar <jar> -testconfigpath <path>) [-r <retention_days>] [-latest yes] [-x local] [-cleanuponly] [-secretDebugCmd] [-t <testcase>] <configfile>
+
+
+while ($ARGV[0] =~ /^-/) {
+ #print $log "DEBUG $0 : ARGV(0)= ".$ARGV[0]."\n";
+ if ($ARGV[0] =~ /^--?l(og)?$/) {
+ shift;
+ $logfile = shift;
+ next;
+ }
+
+ if ($ARGV[0] =~ /^--?t(estgroup)?$/) {
+ shift;
+ push @testgroups, shift;
+ #print $log "DEBUG $0 : TESTGROUP ".$ARGV[0]."\n";
+ next;
+ }
+
+ if ($ARGV[0] =~ /^--?d(escription)?$/) {
+ shift;
+ $testrun_desc = shift;
+ next;
+ }
+
+ if ($ARGV[0] =~ /^--?r(egexp)?$/) {
+ shift;
+ push @testMatches, shift;
+ next;
+ }
+ if ($ARGV[0] =~ /^--?db(log)?$/) {
+ shift;
+ $dblog = shift;
+ next;
+ }
+
+ if ($ARGV[0] =~ /^--?st(artat)?$/) {
+ shift;
+ $startat = shift;
+ next;
+ }
+
+ if ($ARGV[0] =~ /^--?deploycfg$/) {
+ shift;
+ $deploycfg = shift;
+ next;
+ }
+
+ if ($ARGV[0] =~ /^--?deploy$/) {
+ shift;
+ $deploy = 1;
+ next;
+ }
+
+ if ($ARGV[0] =~ /^--?undeploy$/) {
+ shift;
+ $undeploy = 1;
+ next;
+ }
+
+ # Not an argument for us, so just push it into the hash. These arguments
+ # will override values in the config file.
+ my $key = shift;
+ $key =~ s/^--?//;
+ $globalCfg->{$key} = shift;
+}
+
+mkpath( [ $globalCfg->{'localoutpathbase'} ] , 1, 0777) if ( ! -e $globalCfg->{'localoutpathbase'} );
+$globalCfg->{'UID'}= time;
+$logfile = $globalCfg->{'localoutpathbase'} . "/pig_test_harnesss_" . $globalCfg->{'UID'} if $logfile eq "";
+$globalCfg->{'logfile'} = $logfile;
+
+
+my $log;
+open $log, "> $logfile" or die "FATAL ERROR $0 at ".__LINE__." : Can't open $logfile, $!\n";
+
+print "================================================================================================\n";
+print "LOGGING RESULTS TO " . cwd . "/$logfile\n";
+print "================================================================================================\n";
+
+# If they have requested deployment, do it now
+if ($deploy) {
+ if (!$deploycfg) {
+ die "You must define a deployment configuration file using -deploycfg "
+ . "<cfg file> if you want to deploy your test resources.\n";
+ }
+
+ # Read the deployment cfg file
+ print $log "INFO: $0 at ".__LINE__." : Loading configuration file $deploycfg\n";
+ my $cfg = readCfg($deploycfg);
+
+ # Copy the global config into our cfg
+ foreach(keys(%$globalCfg)) {
+ next if $_ eq 'file';
+ $cfg->{$_} = $globalCfg->{$_}; #foreach(keys(%$globalCfg));
+ }
+
+ # Instantiate the TestDeployer
+ my $deployer = TestDeployerFactory::getTestDeployer($cfg);
+ die "FATAL: $0: Deployer does not exist\n" if ( !$deployer );
+
+ eval {
+ $deployer->checkPrerequisites($cfg, $log);
+ };
+ if ($@) {
+ chomp $@;
+ die "Check of prerequites failed: <$@>\n";
+ }
+ eval {
+ $deployer->deploy($cfg, $log);
+ };
+ if ($@) {
+ chomp $@;
+ die "Deployment of test resources failed: <$@>\n";
+ }
+ eval {
+ $deployer->start($cfg, $log);
+ };
+ if ($@) {
+ chomp $@;
+ die "Failed to start test resources: <$@>\n";
+ }
+ eval {
+ $deployer->generateData($cfg, $log);
+ };
+ if ($@) {
+ chomp $@;
+ die "Failed to generate data for testing: <$@>\n";
+ }
+ eval {
+ $deployer->confirmDeployment($cfg, $log);
+ };
+ if ($@) {
+ chomp $@;
+ die "Failed to confirm that test resources were properly deployed: <$@>\n";
+ }
+
+ print $log "INFO: $0 at " . __LINE__ .
+ " : Successfully deployed test resources $deploycfg\n";
+}
+
+# If they said -undeploy test up front that they have a deploycfg file and that we
+# can read it so we lower the risk of running all the tests and then failing to
+# undeploy.
+if ($undeploy) {
+ if (!$deploycfg) {
+ die "You must define a deployment configuration file using -deploycfg "
+ . "<cfg file> if you want to undeploy your test resources.\n";
+ }
+
+ # Read the deployment cfg file
+ print $log "INFO: $0 at ".__LINE__." : Loading configuration file $deploycfg\n";
+ my $cfg = readCfg($deploycfg);
+
+ # Instantiate the TestDeployer
+ my $deployer = TestDeployerFactory::getTestDeployer($cfg);
+ die "FATAL: $0: Deployer does not exist\n" if ( !$deployer );
+}
+
+
+print $log "Beginning test run at " . time . "\n";
+
+my $dbh = undef;
+if($dblog) {
+ # Attach to the database
+ $dbh = new Insert2Mysql($globalCfg->{'dbServer'}, $globalCfg->{'dbDatabase'});
+ $globalCfg->{'trid'} = $dbh->startTestRun($testrun_desc);
+ $dbh->logTestRun($globalCfg->{'trid'}, $logfile);
+
+ # print "Testrun id in database is $globalCfg->{'trid'}\n";
+ print $log "Testrun id $globalCfg->{'trid'}\n";
+}
+
+
+my %testStatuses;
+foreach my $arg (@ARGV) {
+ print $log "INFO: $0 at ".__LINE__." : Loading configuration file $arg\n";
+ my $cfg = readCfg($arg);
+ # Copy contents of global config file into hash.
+ foreach(keys(%$globalCfg)) {
+ next if $_ eq 'file';
+ $cfg->{$_} = $globalCfg->{$_}; # foreach(keys(%$globalCfg));
+ print $log "\nINFO $0: $_=".$cfg->{$_};
+ }
+ print $log "\n";
+
+ my $driver = TestDriverFactory::getTestDriver($cfg);
+ die "FATAL: $0: Driver does not exist\n" if ( !$driver );
+ $driver->run(\@testgroups, \@testMatches, $cfg, $log, $dbh, \%testStatuses, $arg, $startat, $logfile);
+}
+$dbh->endTestRun($globalCfg->{'trid'}) if ($dblog);
+
+# don't remove the space after Final results, it matters.
+TestDriver::printResults(\%testStatuses, \*STDOUT, "Final results ");
+TestDriver::printResults(\%testStatuses, $log, "Final results");
+print $log "Finished test run at " . time . "\n";
+
+# If they have requested undeployment, do it now
+if ($undeploy) {
+ # Read the deployment cfg file
+ print $log "INFO: $0 at ".__LINE__." : Loading configuration file $deploycfg\n";
+ my $cfg = readCfg($deploycfg);
+
+ # Instantiate the TestDeployer
+ my $deployer = TestDeployerFactory::getTestDeployer($cfg);
+ die "FATAL: $0: Deployer does not exist\n" if ( !$deployer );
+
+ eval {
+ $deployer->deleteData($cfg, $log);
+ };
+ if ($@) {
+ chomp $@;
+ warn "Failed to delete data as part of undeploy: <$@>\n";
+ }
+ eval {
+ $deployer->stop($cfg, $log);
+ };
+ if ($@) {
+ chomp $@;
+ warn "Failed to stop test resources: <$@>\n";
+ }
+ eval {
+ $deployer->undeploy($cfg, $log);
+ };
+ if ($@) {
+ chomp $@;
+ warn "Failed to undeploy test resources: <$@>\n";
+ }
+ eval {
+ $deployer->confirmUndeployment($cfg, $log);
+ };
+ if ($@) {
+ chomp $@;
+ die "Failed to confirm that test resources were properly undeployed: <$@>\n";
+ }
+
+ print $log "INFO: $0 at " . __LINE__ .
+ " : Successfully undeployed test resources $deploycfg\n";
+}
+close $log;
+
+exit exitStatus(\%testStatuses);
+
Modified: pig/trunk/test/e2e/pig/build.xml
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/build.xml?rev=1159790&r1=1159789&r2=1159790&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/build.xml (original)
+++ pig/trunk/test/e2e/pig/build.xml Fri Aug 19 21:15:46 2011
@@ -41,7 +41,7 @@
<property name="tool.src" value="${basedir}/tools"/>
<property name="harness.dir" value="${basedir}/../harness"/>
- <property name="harness.tar" value="${harness.dir}/pigharness.tar"/>
+ <property name="harness.tar" value="${harness.dir}/harness.tar"/>
<property name="test.location" value="${basedir}/testdist"/>
<property name="benchmark.location" value="${test.location}/benchmarks"/>
@@ -129,10 +129,12 @@
<!-- Check that the necessary properties are setup -->
<target name="property-check">
- <fail message="Please set the property pig.harness.old.pig to the directory where your old version of Pig is installed"
- unless="pig.harness.old.pig"/>
- <fail message="Please set the property pig.harness.cluster to the directory of your hadoop installation"
- unless="pig.harness.cluster"/>
+ <fail message="Please set the property harness.old.pig to the directory where your old version of Pig is installed"
+ unless="harness.old.pig"/>
+ <fail message="Please set the property harness.cluster.conf to the conf directory of your hadoop installation"
+ unless="harness.cluster.conf"/>
+ <fail message="Please set the property harness.cluster.bin to the binary executable of your hadoop installation"
+ unless="harness.cluster.bin"/>
</target>
<!-- Prep the test area -->
@@ -145,7 +147,7 @@
<chmod perm="ugo+x" type="file">
<fileset dir="${test.location}/libexec" />
- <fileset file="${test.location}/pig_test_harness.pl"/>
+ <fileset file="${test.location}/test_harness.pl"/>
</chmod>
</target>
@@ -155,14 +157,15 @@
<!-- If they have not specified tests to run then null it out -->
<property name="tests.to.run" value=""/>
- <exec executable="./pig_test_harness.pl" dir="${test.location}" failonerror="true">
- <env key="PIG_HARNESS_ROOT" value="."/>
+ <exec executable="./test_harness.pl" dir="${test.location}" failonerror="true">
+ <env key="HARNESS_ROOT" value="."/>
<env key="PH_LOCAL" value="."/>
<env key="PH_OUT" value="."/>
<env key="PH_ROOT" value="."/>
<env key="PH_PIG" value="${pig.dir}"/>
- <env key="PH_OLDPIG" value="${pig.harness.old.pig}"/>
- <env key="PH_CLUSTER" value="${pig.harness.cluster}"/>
+ <env key="PH_OLDPIG" value="${harness.old.pig}"/>
+ <env key="PH_CLUSTER_CONF" value="${harness.cluster.conf}"/>
+ <env key="PH_CLUSTER_BIN" value="${harness.cluster.bin}"/>
<env key="PH_JYTHON_JAR" value="${jython.jar}"/>
<arg line="${tests.to.run}"/>
<arg value="${test.location}/tests/cmdline.conf"/>
@@ -183,14 +186,15 @@
</target>
<target name="deploy-base" depends="property-check, tar, init-test, init-deploy">
- <exec executable="./pig_test_harness.pl" dir="${test.location}" failonerror="true">
- <env key="PIG_HARNESS_ROOT" value="."/>
+ <exec executable="./test_harness.pl" dir="${test.location}" failonerror="true">
+ <env key="HARNESS_ROOT" value="."/>
<env key="PH_LOCAL" value="."/>
<env key="PH_OUT" value="."/>
<env key="PH_ROOT" value="."/>
<env key="PH_PIG" value="${pig.dir}"/>
- <env key="PH_OLDPIG" value="${pig.harness.old.pig}"/>
- <env key="PH_CLUSTER" value="${pig.harness.cluster}"/>
+ <env key="PH_OLDPIG" value="${harness.old.pig}"/>
+ <env key="PH_CLUSTER_CONF" value="${harness.cluster.conf}"/>
+ <env key="PH_CLUSTER_BIN" value="${harness.cluster.bin}"/>
<env key="PH_JYTHON_JAR" value="${jython.jar}"/>
<arg value="-deploycfg"/>
Modified: pig/trunk/test/e2e/pig/conf/default.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/conf/default.conf?rev=1159790&r1=1159789&r2=1159790&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/conf/default.conf (original)
+++ pig/trunk/test/e2e/pig/conf/default.conf Fri Aug 19 21:15:46 2011
@@ -19,11 +19,12 @@ chomp $me;
# The contents of this file can be rewritten to fit your installation.
# Also, you can define the following environment variables and set things up as in the test setup
-# PH_ROOT Root directory where test harness is installed
-# PH_LOCAL Root directory for input and output for local mode tests
-# PH_OUT Root directory where output data will be stored (on local disk, not HDFS)
-# PH_CLUSTER Root directory for cluster being used
-# PH_PIG Root directory for Pig version being used
+# PH_ROOT Root directory where test harness is installed
+# PH_LOCAL Root directory for input and output for local mode tests
+# PH_OUT Root directory where output data will be stored (on local disk, not HDFS)
+# PH_CLUSTER_BIN Conf directory for cluster being used
+# PH_CLUSTER_CONF Binary executable for cluster being used
+# PH_PIG Root directory for Pig version being used
$cfg = {
#HDFS
@@ -40,17 +41,17 @@ $cfg = {
, 'benchmarkPath' => "$ENV{PH_OUT}/benchmarks"
, 'scriptPath' => "$ENV{PH_ROOT}/libexec"
, 'tmpPath' => '/tmp/pigtest'
- , 'jythonjar' => "$ENV{PH_JYTHON_JAR}"
+ , 'jythonjar' => "$ENV{PH_JYTHON_JAR}"
#PIG
- , 'testconfigpath' => "$ENV{PH_CLUSTER}/conf/"
+ , 'testconfigpath' => "$ENV{PH_CLUSTER_CONF}"
, 'funcjarPath' => "$ENV{PH_ROOT}/lib/java"
, 'paramPath' => "$ENV{PH_ROOT}/paramfiles"
, 'pigpath' => "$ENV{PH_PIG}"
- , 'oldpigpath' => "$ENV{PH_OLDPIG}"
+ , 'oldpigpath' => "$ENV{PH_OLDPIG}"
- #HADOOP
- , 'hadoopHome' => "$ENV{PH_ROOT}/lib"
+ #HADOOP
+ , 'hadoopHome' => "$ENV{PH_ROOT}/lib"
#DATABASE
@@ -66,7 +67,7 @@ $cfg = {
,'local.bin' => '/usr/bin'
,'logDir' => "$ENV{PH_OUT}/log"
- ,'propertiesFile' => "./conf/testpropertiesfile.conf"
- ,'pigharness.console.level' => 'ERROR'
+ ,'propertiesFile' => "./conf/testpropertiesfile.conf"
+ ,'harness.console.level' => 'ERROR'
};
Modified: pig/trunk/test/e2e/pig/conf/existing_deployer.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/conf/existing_deployer.conf?rev=1159790&r1=1159789&r2=1159790&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/conf/existing_deployer.conf (original)
+++ pig/trunk/test/e2e/pig/conf/existing_deployer.conf Fri Aug 19 21:15:46 2011
@@ -25,7 +25,8 @@ $cfg = {
'deployer' => 'ExistingClusterDeployer',
# hadoop values
- 'hadoopdir' => $ENV{'PH_CLUSTER'},
+ 'hadoopconfdir' => $ENV{'PH_CLUSTER_CONF'},
+ 'hadoopbin' => $ENV{'PH_CLUSTER_BIN'},
# db values
# 'dbuser' => 'pigtester',
Modified: pig/trunk/test/e2e/pig/conf/testpropertiesfile.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/conf/testpropertiesfile.conf?rev=1159790&r1=1159789&r2=1159790&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/conf/testpropertiesfile.conf (original)
+++ pig/trunk/test/e2e/pig/conf/testpropertiesfile.conf Fri Aug 19 21:15:46 2011
@@ -15,7 +15,7 @@
# limitations under the License.
$cfg = {
- 'pigharness.log' => './out/harness.log'
- ,'pigharness.log.level' => 'DEBUG'
- ,'pigharness.console.level'=> 'INFO'
+ 'harness.log' => './out/harness.log'
+ ,'harness.log.level' => 'DEBUG'
+ ,'harness.console.level'=> 'INFO'
};
Modified: pig/trunk/test/e2e/pig/deployers/ExistingClusterDeployer.pm
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/deployers/ExistingClusterDeployer.pm?rev=1159790&r1=1159789&r2=1159790&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/deployers/ExistingClusterDeployer.pm (original)
+++ pig/trunk/test/e2e/pig/deployers/ExistingClusterDeployer.pm Fri Aug 19 21:15:46 2011
@@ -63,11 +63,18 @@ sub checkPrerequisites
{
my ($self, $cfg, $log) = @_;
- # They must have declared the directory for their Hadoop installation
- if (! defined $cfg->{'hadoopdir'} || $cfg->{'hadoopdir'} eq "") {
- print $log "You must set the key 'hadoopdir' to your Hadoop directory "
+ # They must have declared the conf directory for their Hadoop installation
+ if (! defined $cfg->{'hadoopconfdir'} || $cfg->{'hadoopconfdir'} eq "") {
+ print $log "You must set the key 'hadoopconfdir' to your Hadoop conf directory "
. "in existing.conf\n";
- die "hadoopdir is not set in existing.conf\n";
+ die "hadoopconfdir is not set in existing.conf\n";
+ }
+
+ # They must have declared the executable path for their Hadoop installation
+ if (! defined $cfg->{'hadoopbin'} || $cfg->{'hadoopbin'} eq "") {
+ print $log "You must set the key 'hadoopbin' to your Hadoop bin path"
+ . "in existing.conf\n";
+ die "hadoopbin is not set in existing.conf\n";
}
# Run a quick and easy Hadoop command to make sure we can
@@ -314,9 +321,9 @@ sub runHadoopCmd($$$$)
my ($self, $cfg, $log, $c) = @_;
# set the PIG_CLASSPATH environment variable
- $ENV{'HADOOP_CLASSPATH'} = "$cfg->{'hadoopdir'}/conf";
+ $ENV{'HADOOP_CLASSPATH'} = "$cfg->{'hadoopconfdir'}";
- my @cmd = ("$cfg->{'hadoopdir'}/bin/hadoop");
+ my @cmd = ("$cfg->{'hadoopbin'}");
push(@cmd, split(' ', $c));
$self->runCmd($log, \@cmd);
Modified: pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm?rev=1159790&r1=1159789&r2=1159790&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm (original)
+++ pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm Fri Aug 19 21:15:46 2011
@@ -32,7 +32,7 @@ use English;
our $className= "TestDriver";
our @ISA = "$className";
-our $ROOT = (defined $ENV{'PIG_HARNESS_ROOT'} ? $ENV{'PIG_HARNESS_ROOT'} : die "ERROR: You must set environment variable PIG_HARNESS_ROOT\n");
+our $ROOT = (defined $ENV{'HARNESS_ROOT'} ? $ENV{'HARNESS_ROOT'} : die "ERROR: You must set environment variable HARNESS_ROOT\n");
our $toolpath = "$ROOT/libexec/PigTest";
my $passedStr = 'passed';
@@ -102,7 +102,7 @@ sub replaceParameters
$cmd =~ s/:OUTPATH:/$outfile/g;
# $ENV
- $cmd =~ s/:PIGHARNESS:/$ENV{PIG_HARNESS_ROOT}/g;
+ $cmd =~ s/:PIGHARNESS:/$ENV{HARNESS_ROOT}/g;
# $testCmd
$cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g;