You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2011/04/13 06:29:39 UTC
svn commit: r1091642 - in /pig/trunk: ./ test/e2e/harness/
test/e2e/harness/test/tests/ test/e2e/pig/
Author: gates
Date: Wed Apr 13 04:29:39 2011
New Revision: 1091642
URL: http://svn.apache.org/viewvc?rev=1091642&view=rev
Log:
PIG-1954 Design deployment interface for e2e test harness
Added:
pig/trunk/test/e2e/harness/TestDeployer.pm
pig/trunk/test/e2e/harness/TestDeployerFactory.pm
pig/trunk/test/e2e/harness/TestDeployerTest.pm
pig/trunk/test/e2e/harness/test/tests/deploy.conf
Modified:
pig/trunk/CHANGES.txt
pig/trunk/test/e2e/harness/Makefile
pig/trunk/test/e2e/harness/pig_test_harness.pl
pig/trunk/test/e2e/harness/test/tests/test.conf
pig/trunk/test/e2e/pig/Makefile
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1091642&r1=1091641&r2=1091642&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Apr 13 04:29:39 2011
@@ -32,6 +32,8 @@ PIG-1876: Typed map for Pig (daijy)
IMPROVEMENTS
+PIG-1954: Design deployment interface for e2e test harness (gates)
+
PIG-1881: Need a special interface for Penny (Inspector Gadget) (laukik via
gates)
Modified: pig/trunk/test/e2e/harness/Makefile
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/harness/Makefile?rev=1091642&r1=1091641&r2=1091642&view=diff
==============================================================================
--- pig/trunk/test/e2e/harness/Makefile (original)
+++ pig/trunk/test/e2e/harness/Makefile Wed Apr 13 04:29:39 2011
@@ -17,6 +17,8 @@
PIG_TH_DIR = .
PIG_TH_MODULES = $(PIG_TH_DIR)/Insert2Mysql.pm \
+ $(PIG_TH_DIR)/TestDeployerFactory.pm \
+ $(PIG_TH_DIR)/TestDeployer.pm \
$(PIG_TH_DIR)/TestDriverFactory.pm \
$(PIG_TH_DIR)/TestDriver.pm \
$(PIG_TH_DIR)/TestReport.pm \
@@ -25,7 +27,8 @@ PIG_TH_MODULES = $(PIG_TH_DIR)/Insert2My
PIG_TH_TEST_DIR = $(PIG_TH_DIR)
-PIG_TH_TEST_MODULES = $(PIG_TH_TEST_DIR)/TestDriverTest.pm
+PIG_TH_TEST_MODULES = $(PIG_TH_TEST_DIR)/TestDriverTest.pm \
+ $(PIG_TH_TEST_DIR)/TestDeployerTest.pm
ARCHIVE = pigharness.tar
@@ -40,12 +43,13 @@ test: $(ARCHIVE)
mkdir -p $(TEST_DIR)
mkdir -p $(TEST_DIR)/conf
cp $(ARCHIVE) $(TEST_DIR)
- cp test/tests/test.conf $(TEST_DIR)
+ cp test/tests/*.conf $(TEST_DIR)
cp test/conf/default.conf $(TEST_DIR)/conf
(cd $(TEST_DIR); tar xf $(ARCHIVE))
mkdir -p $(TEST_DIR)/$(PIG_TH_TEST_DIR)
cp $(PIG_TH_TEST_MODULES) $(TEST_DIR)/$(PIG_TH_TEST_DIR)
(cd $(TEST_DIR); export PIG_HARNESS_ROOT=.; ./$(MAIN) test.conf)
+ (cd $(TEST_DIR); export PIG_HARNESS_ROOT=.; ./$(MAIN) -deploycfg deploy.conf -deploy -undeploy test.conf)
clean:
rm -f $(ARCHIVE)
Added: pig/trunk/test/e2e/harness/TestDeployer.pm
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/harness/TestDeployer.pm?rev=1091642&view=auto
==============================================================================
--- pig/trunk/test/e2e/harness/TestDeployer.pm (added)
+++ pig/trunk/test/e2e/harness/TestDeployer.pm Wed Apr 13 04:29:39 2011
@@ -0,0 +1,217 @@
+############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package TestDeployer;
+
+###########################################################################
+# Class: TestDeployer
+# A base class for TestDeployer. This interface defines a set of functions
+# that can be called to deploy and undeploy resources for testing. By default
+# these methods are not called. If the user adds -deploy to the command line
+# then the following methods will be called on this interface, in the following
+# order:
+# checkPrerequisites()
+# deploy()
+# start()
+# generateData()
+# confirmDeployment()
+# If the user adds -undeploy to the command line then the following methods
+# will be called on this inteface, in the following order:
+# deleteData()
+# stop()
+# undeploy()
+# confirmUndeployment()
+#
+# If either -deploy or -undeploy are invoked then -deploycfg <deply cfg file> must
+# be supplied too.
+
+##############################################################################
+# Sub: new
+# Constructor
+#
+# Paramaters:
+# None
+#
+# Returns:
+# None.
+sub new
+{
+ my $proto = shift;
+ my $class = ref($proto) || $proto;
+ my $self = {};
+
+ bless($self, $class);
+
+ return $self;
+}
+
+##############################################################################
+# Sub: checkPrerequisites
+# Check any prerequisites before a deployment is begun. For example if a
+# particular deployment required the use of a database system it could
+# check here that the db was installed and accessible.
+#
+# Paramaters:
+# globalHash - hash from config file, including deployment config
+# log - log file handle
+#
+# Returns:
+# None
+#
+sub checkPrerequisites
+{
+}
+
+##############################################################################
+# Sub: deploy
+# Deploy any required packages
+#
+# Paramaters:
+# globalHash - hash from config file, including deployment config
+# log - log file handle
+#
+# Returns:
+# None
+#
+sub deploy
+{
+}
+
+##############################################################################
+# Sub: start
+# Start any software modules that are needed.
+#
+# Paramaters:
+# globalHash - hash from config file, including deployment config
+# log - log file handle
+#
+# Returns:
+# None
+#
+sub start
+{
+}
+
+##############################################################################
+# Sub: generateData
+# Generate any data needed for this test run.
+#
+# Paramaters:
+# globalHash - hash from config file, including deployment config
+# log - log file handle
+#
+# Returns:
+# None
+#
+sub generateData
+{
+}
+
+##############################################################################
+# Sub: confirmDeployment
+# Run checks to confirm that the deployment was successful. When this is
+# done the testing environment should be ready to run.
+#
+# Paramaters:
+# globalHash - hash from config file, including deployment config
+# log - log file handle
+#
+# Returns:
+# Nothing
+# This method should die with an appropriate error message if there is
+# an issue.
+#
+sub confirmDeployment
+{
+ die "$0 INFO : confirmDeployment is a virtual function!";
+}
+
+##############################################################################
+# Sub: deleteData
+# Remove any data created that will not be removed by undeploying.
+#
+# Paramaters:
+# globalHash - hash from config file, including deployment config
+# log - log file handle
+#
+# Returns:
+# None
+#
+sub deleteData
+{
+}
+
+##############################################################################
+# Sub: stop
+# Stop any servers or systems that are no longer needed once testing is
+# completed.
+#
+# Paramaters:
+# globalHash - hash from config file, including deployment config
+# log - log file handle
+#
+# Returns:
+# None
+#
+sub stop
+{
+}
+
+##############################################################################
+# Sub: undeploy
+# Remove any packages that were installed as part of the deployment.
+#
+# Paramaters:
+# globalHash - hash from config file, including deployment config
+# log - log file handle
+#
+# Returns:
+# None
+#
+sub undeploy
+{
+}
+
+##############################################################################
+# Sub: confirmUndeployment
+# Run checks to confirm that the undeployment was successful. When this is
+# done anything that must be turned off or removed should be turned off or
+# removed.
+#
+# Paramaters:
+# globalHash - hash from config file, including deployment config
+# log - log file handle
+#
+# Returns:
+# Nothing
+# This method should die with an appropriate error message if there is
+# an issue.
+#
+sub confirmUndeployment
+{
+ die "$0 INFO : confirmUndeployment is a virtual function!";
+}
+
+1;
+
+
+
+
+
+
+
+
+
Added: pig/trunk/test/e2e/harness/TestDeployerFactory.pm
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/harness/TestDeployerFactory.pm?rev=1091642&view=auto
==============================================================================
--- pig/trunk/test/e2e/harness/TestDeployerFactory.pm (added)
+++ pig/trunk/test/e2e/harness/TestDeployerFactory.pm Wed Apr 13 04:29:39 2011
@@ -0,0 +1,56 @@
+############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package TestDeployerFactory;
+
+###############################################################################
+# Class: TestDeployerFactory
+# A factory for TestDeployers. This will read the environment and return the
+# correct TestDeployer.
+#
+
+use strict;
+
+require Exporter;
+our @ISA = "Exporter";
+our @EXPORT_OK = qw(splitLine readLine isTag);
+
+###############################################################################
+# Sub: getTestDeployer
+# Returns the appropriate Test Deployer. This is determined by reading
+# the 'deployer' value in the config file.
+#
+# Returns:
+# instance of appropriate subclass of TestDeployer
+#
+sub getTestDeployer
+{
+ my $cfg = shift;
+
+ if (not defined $cfg->{'deployer'}) {
+ die "$0 FATAL : I didn't see a deployer key in the file, I don't know "
+ . "what deployer to instantiate.\n";
+ }
+
+ my $className = $cfg->{'deployer'};
+
+ require "$className.pm";
+ my $deployer = new $className();
+
+ return $deployer;
+}
+
+1;
Added: pig/trunk/test/e2e/harness/TestDeployerTest.pm
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/harness/TestDeployerTest.pm?rev=1091642&view=auto
==============================================================================
--- pig/trunk/test/e2e/harness/TestDeployerTest.pm (added)
+++ pig/trunk/test/e2e/harness/TestDeployerTest.pm Wed Apr 13 04:29:39 2011
@@ -0,0 +1,57 @@
+############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package TestDeployerTest;
+
+use TestDeployer;
+
+our @ISA = "TestDeployer";
+
+use strict;
+
+###########################################################################
+# Class: TestDeployerTest
+# A very simple test of TestDeployer. It does nothing.
+
+##############################################################################
+# Sub: new
+# Constructor
+#
+# Paramaters:
+# None
+#
+# Returns:
+# None.
+sub new
+{
+ my $proto = shift;
+ my $class = ref($proto) || $proto;
+ my $self = {};
+
+ bless($self, $class);
+
+ return $self;
+}
+
+sub confirmDeployment
+{
+}
+
+sub confirmUndeployment
+{
+}
+
+1;
Modified: pig/trunk/test/e2e/harness/pig_test_harness.pl
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/harness/pig_test_harness.pl?rev=1091642&r1=1091641&r2=1091642&view=diff
==============================================================================
--- pig/trunk/test/e2e/harness/pig_test_harness.pl (original)
+++ pig/trunk/test/e2e/harness/pig_test_harness.pl Wed Apr 13 04:29:39 2011
@@ -82,6 +82,8 @@ unshift( @INC, ".");
require TestDriver;
require TestDriverFactory;
+require TestDeployer;
+require TestDeployerFactory;
require Insert2Mysql;
require Properties;
require Log; # why aren't we using log4perl?
@@ -98,14 +100,20 @@ our $dblog;
# usage string
sub usage
{
- return "Usage: $0 [OPTIONS] conffile [... confile]\n
- OPTIONS:\n
- -l <log file name> - set log file name\n
- -t <test group name> - set test group testcases\n
- -d <description name> - set description for MySQL database\n
- -r <regexp> - set regular expression for test group testcases\n
- -db <0 or 1> - disable using MySQL database if set to 0\n
- -st <group name> - start test from provided group name\n";
+ return
+"Usage: $0 [OPTIONS] conffile [... confile]
+ OPTIONS:
+ -l <log file name> - set log file name
+ -t <test group name> - set test group testcases
+ -d <description name> - set description for MySQL database
+ -r <regexp> - set regular expression for test group testcases
+ -db <0 or 1> - disable using MySQL database if set to 0
+ -st <group name> - start test from provided group name
+ -deploycfg <deploy cfg file> -deploy - Deploy the test setup before testing
+ <deploy cfg file> is the configuration file for deployment
+ -deploycfg <deploy cfg file> -undeploy - Undeploy the test setup after testing
+ <deploy cfg file> is the configuration file for deployment
+ ";
}
##############################################################################
@@ -240,6 +248,9 @@ my $testrun_desc = 'none';
my @testgroups;
my @testMatches;
my $startat = undef;
+my $deploycfg = undef;
+my $deploy = undef;
+my $undeploy = undef;
my $help=0;
die usage() if (@ARGV == 0);
@@ -323,6 +334,24 @@ while ($ARGV[0] =~ /^-/) {
next;
}
+ if ($ARGV[0] =~ /^--?deploycfg$/) {
+ shift;
+ $deploycfg = shift;
+ next;
+ }
+
+ if ($ARGV[0] =~ /^--?deploy$/) {
+ shift;
+ $deploy = 1;
+ next;
+ }
+
+ if ($ARGV[0] =~ /^--?undeploy$/) {
+ shift;
+ $undeploy = 1;
+ next;
+ }
+
# Not an argument for us, so just push it into the hash. These arguments
# will override values in the config file.
my $key = shift;
@@ -343,6 +372,80 @@ print "=================================
print "LOGGING RESULTS TO $logfile\n";
print "================================================================================================\n";
+# If they have requested deployment, do it now
+if ($deploy) {
+ if (!$deploycfg) {
+ die "You must define a deployment configuration file using -deploycfg "
+ . "<cfg file> if you want to deploy your test resources.\n";
+ }
+
+ # Read the deployment cfg file
+ print $log "INFO: $0 at ".__LINE__." : Loading configuration file $deploycfg\n";
+ my $cfg = readCfg($deploycfg);
+
+ # Instantiate the TestDeployer
+ my $deployer = TestDeployerFactory::getTestDeployer($cfg);
+ die "FATAL: $0: Deployer does not exist\n" if ( !$deployer );
+
+ eval {
+ $deployer->checkPrerequisites($cfg, $log);
+ };
+ if ($@) {
+ chomp $@;
+ die "Check of prerequites failed: <$@>\n";
+ }
+ eval {
+ $deployer->deploy($cfg, $log);
+ };
+ if ($@) {
+ chomp $@;
+ die "Deployment of test resources failed: <$@>\n";
+ }
+ eval {
+ $deployer->start($cfg, $log);
+ };
+ if ($@) {
+ chomp $@;
+ die "Failed to start test resources: <$@>\n";
+ }
+ eval {
+ $deployer->generateData($cfg, $log);
+ };
+ if ($@) {
+ chomp $@;
+ die "Failed to generate data for testing: <$@>\n";
+ }
+ eval {
+ $deployer->confirmDeployment($cfg, $log);
+ };
+ if ($@) {
+ chomp $@;
+ die "Failed to confirm that test resources were properly deployed: <$@>\n";
+ }
+
+ print $log "INFO: $0 at " . __LINE__ .
+ " : Successfully deployed test resources $deploycfg\n";
+}
+
+# If they said -undeploy test up front that they have a deploycfg file and that we
+# can read it so we lower the risk of running all the tests and then failing to
+# undeploy.
+if ($undeploy) {
+ if (!$deploycfg) {
+ die "You must define a deployment configuration file using -deploycfg "
+ . "<cfg file> if you want to undeploy your test resources.\n";
+ }
+
+ # Read the deployment cfg file
+ print $log "INFO: $0 at ".__LINE__." : Loading configuration file $deploycfg\n";
+ my $cfg = readCfg($deploycfg);
+
+ # Instantiate the TestDeployer
+ my $deployer = TestDeployerFactory::getTestDeployer($cfg);
+ die "FATAL: $0: Deployer does not exist\n" if ( !$deployer );
+}
+
+
print $log "Beginning test run at " . time . "\n";
my $dbh = undef;
@@ -359,7 +462,7 @@ if($dblog) {
my %testStatuses;
foreach my $arg (@ARGV) {
- print $log "INFO: $0 at ".__LINE__." : Loading configuration file $arg\n";
+ print $log "INFO: $0 at ".__LINE__." : Loading configuration file $arg\n";
my $cfg = readCfg($arg);
# Copy contents of global config file into hash.
foreach(keys(%$globalCfg)) {
@@ -379,7 +482,49 @@ $dbh->endTestRun($globalCfg->{'trid'}) i
TestDriver::printResults(\%testStatuses, \*STDOUT, "Final results ");
TestDriver::printResults(\%testStatuses, $log, "Final results");
print $log "Finished test run at " . time . "\n";
-
+
+# If they have requested undeployment, do it now
+if ($undeploy) {
+ # Read the deployment cfg file
+ print $log "INFO: $0 at ".__LINE__." : Loading configuration file $deploycfg\n";
+ my $cfg = readCfg($deploycfg);
+
+ # Instantiate the TestDeployer
+ my $deployer = TestDeployerFactory::getTestDeployer($cfg);
+ die "FATAL: $0: Deployer does not exist\n" if ( !$deployer );
+
+ eval {
+ $deployer->deleteData($cfg, $log);
+ };
+ if ($@) {
+ chomp $@;
+ warn "Failed to delete data as part of undeploy: <$@>\n";
+ }
+ eval {
+ $deployer->stop($cfg, $log);
+ };
+ if ($@) {
+ chomp $@;
+ warn "Failed to stop test resources: <$@>\n";
+ }
+ eval {
+ $deployer->undeploy($cfg, $log);
+ };
+ if ($@) {
+ chomp $@;
+ warn "Failed to undeploy test resources: <$@>\n";
+ }
+ eval {
+ $deployer->confirmUndeployment($cfg, $log);
+ };
+ if ($@) {
+ chomp $@;
+ die "Failed to confirm that test resources were properly undeployed: <$@>\n";
+ }
+
+ print $log "INFO: $0 at " . __LINE__ .
+ " : Successfully undeployed test resources $deploycfg\n";
+}
close $log;
exit exitStatus(\%testStatuses);
Added: pig/trunk/test/e2e/harness/test/tests/deploy.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/harness/test/tests/deploy.conf?rev=1091642&view=auto
==============================================================================
--- pig/trunk/test/e2e/harness/test/tests/deploy.conf (added)
+++ pig/trunk/test/e2e/harness/test/tests/deploy.conf Wed Apr 13 04:29:39 2011
@@ -0,0 +1,28 @@
+############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+###############################################################################
+# Test for TestHarness itself.
+#
+#
+
+$cfg = {
+ 'deployer' => 'TestDeployerTest',
+}
+;
+
+
+
Modified: pig/trunk/test/e2e/harness/test/tests/test.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/harness/test/tests/test.conf?rev=1091642&r1=1091641&r2=1091642&view=diff
==============================================================================
--- pig/trunk/test/e2e/harness/test/tests/test.conf (original)
+++ pig/trunk/test/e2e/harness/test/tests/test.conf Wed Apr 13 04:29:39 2011
@@ -15,7 +15,7 @@
# limitations under the License.
###############################################################################
-# Test for TestHarness itself. The result should be 2 passes and 1 failure
+# Test for TestHarness itself. The result should be 2 passes
#
#
@@ -39,16 +39,6 @@ $cfg = {
}
]
},
- {
- 'name' => 'failing',
- 'tests' => [
- {
- 'num' => 1,
- 'rc' => 1,
- 'benchmark_rc' => 2
- },
- ]
- },
],
}
;
Modified: pig/trunk/test/e2e/pig/Makefile
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/Makefile?rev=1091642&r1=1091641&r2=1091642&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/Makefile (original)
+++ pig/trunk/test/e2e/pig/Makefile Wed Apr 13 04:29:39 2011
@@ -107,15 +107,18 @@ $(ARCHIVE): $(TESTS) $(DRIVERS) $(CONF_F
build_udfs:
(if [ "$${PH_PIG}x" == "x" ] ; then echo \
"You must set the environment variable PH_PIG" \
+ "to the directory your pig.jar is in " \
"before building the UDFs"; 1; fi)
(cd $(JAVA_UDF_DIR); ant -Dpig.jarfile=$${PH_PIG}/pig.jar)
test: $(ARCHIVE) build_udfs
(if [ "$${PH_CLUSTER}x" == "x" ] ; then echo \
- "You must set the environment variable PH_CLUSTER " \
+ "You must set the environment variable PH_CLUSTER" \
+ "to the directory that contains your hadoop-site.xml" \
"before running the tests"; 1; fi)
(if [ "$${PH_JYTHON_JAR}x" == "x" ] ; then echo \
- "You must set the environment variable PH_JYTHON_JAR " \
+ "You must set the environment variable PH_JYTHON_JAR" \
+ "to the path of your jython jar" \
"before running the tests"; 1; fi)
mkdir -p $(TEST_DIST_DIR)/benchmarks
cp $(ARCHIVE) $(TEST_DIST_DIR)