You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2011/03/31 17:31:36 UTC
svn commit: r1087340 [2/9] - in /pig/trunk: ./ test/e2e/ test/e2e/harness/
test/e2e/harness/test/ test/e2e/harness/test/conf/
test/e2e/harness/test/tests/ test/e2e/pig/ test/e2e/pig/conf/
test/e2e/pig/drivers/ test/e2e/pig/lib/ test/e2e/pig/paramfiles/...
Added: pig/trunk/test/e2e/pig/Makefile
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/Makefile?rev=1087340&view=auto
==============================================================================
--- pig/trunk/test/e2e/pig/Makefile (added)
+++ pig/trunk/test/e2e/pig/Makefile Thu Mar 31 15:31:33 2011
@@ -0,0 +1,148 @@
+############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DRIVER_DIR = ./drivers
+
+DRIVERS = $(DRIVER_DIR)/TestDriverPig.pm $(DRIVER_DIR)/Util.pm
+
+TEST_DIR = ./tests
+
+TESTS = $(TEST_DIR)/bigdata.conf $(TEST_DIR)/cmdline.conf \
+ $(TEST_DIR)/grunt.conf \
+ $(TEST_DIR)/multiquery.conf $(TEST_DIR)/negative.conf \
+ $(TEST_DIR)/nightly.conf $(TEST_DIR)/streaming.conf \
+ $(TEST_DIR)/turing_jython.conf
+
+CONF_DIR = ./conf
+
+CONF_FILES = $(CONF_DIR)/default.conf
+
+TOOL_DIR = ./tools
+
+TEST_TOOL_DIR = $(TOOL_DIR)/test
+
+GEN_TOOL_DIR = $(TOOL_DIR)/generate
+
+TOOLS = $(TEST_TOOL_DIR)/explainchk.pl $(TEST_TOOL_DIR)/floatpostprocessor.pl
+
+ARCHIVE = pigtests.tar
+
+TEST_DIST_DIR = testdist
+
+TESTS_TO_RUN ?= -t Checkin -t LoaderDefaultDir -t LoaderPigStorageArg \
+ -t FilterBoolean -t FilterEq -t FilterMatches -t FilterUdf \
+ -t CoGroupFlatten -t Join -t Distinct -t Cross -t Union \
+ -t Glob -t Arithmetic -t Regression -t Unicode -t Parameters \
+ -t ImplicitSplit -t describe -t MissingColumns -t Aliases \
+ -t Lineage -t Casts -t ClassResolution \
+ -t MergeJoin -t SkewedJoin -t CollectedGroup -t Accumulator \
+ -t PruneColumns -t Bzip -t Scalar -t Scripting -t NestedDescribe \
+ -t UdfDistributedCache -t Describe_cmdline -t Unicode_cmdline \
+ -t Warning -t NoSuchFile -t BadFunc -t FileExists \
+ -t NegForeach -t NotAllowed -t UdfException \
+ -t LineageErrors -t ComputeSpec -t JobManagement \
+ -t StreamingIO -t NonStreaming -t StreamingPerformance \
+ -t RaceConditions -t StreamingLocal -t Jython_Checkin
+
+
+
+# Not yet working:
+# LoaderBinStorage - results doctored
+# LoaderTestLoader - results doctored
+# GroupAggFunc - results doctored
+# EvalFunc - results doctored
+# CoGroup - results doctored
+# Foreach - results doctored
+# Order - results doctored
+# Bincond - results doctored
+# Types - results doctored
+# Limit - results doctored
+# Split - results doctored
+# SecondarySort - results doctored
+# Native - results doctored
+# Partitioner - issues unclear
+# CastScalar - issues unclear
+# udf_TOBAGandTOTUPLE -results doctored
+# MergeOperator - issues unclear
+# Grunt - needs reworked with different inputs
+# MultiQuery - results doctored
+# SyntaxErrors - needs streaming resources
+# CmdErrors - needs streaming resources
+# StreamingErrors - needs streaming resources
+# IOErrors - needs streaming resources
+# StreamingLocalErrors - needs streaming resources
+# Jython_* - needs switch from java to bin/pig
+
+UDF_DIR = udfs
+
+JAVA_UDF_DIR = $(UDF_DIR)/java
+
+UDF_JAR = $(JAVA_UDF_DIR)/testudf.jar
+
+PYTHON_UDF_DIR = $(UDF_DIR)/python
+
+PARAM_FILES = paramfiles/params_3
+
+LIB_DIR = lib
+
+STREAMING_DIR = streaming
+
+$(ARCHIVE): $(TESTS) $(DRIVERS) $(CONF_FILES) $(TOOLS)
+ tar cf $(ARCHIVE) $(DRIVERS) $(TESTS) $(CONF_FILES) $(TOOLS)
+
+# Note, you must set PH_PIG_JARFILE environment variable
+build_udfs:
+ (if [ "$${PH_PIG}x" == "x" ] ; then echo \
+ "You must set the environment variable PH_PIG" \
+ "before building the UDFs"; 1; fi)
+ (cd $(JAVA_UDF_DIR); ant -Dpig.jarfile=$${PH_PIG}/pig.jar)
+
+test: $(ARCHIVE) build_udfs
+ (if [ "$${PH_CLUSTER}x" == "x" ] ; then echo \
+ "You must set the environment variable PH_CLUSTER " \
+ "before running the tests"; 1; fi)
+ (if [ "$${PH_JYTHON_JAR}x" == "x" ] ; then echo \
+ "You must set the environment variable PH_JYTHON_JAR " \
+ "before running the tests"; 1; fi)
+ mkdir -p $(TEST_DIST_DIR)/benchmarks
+ cp $(ARCHIVE) $(TEST_DIST_DIR)
+ cp ../harness/pigharness.tar $(TEST_DIST_DIR)
+ (cd $(TEST_DIST_DIR); tar xf $(ARCHIVE))
+ (cd $(TEST_DIST_DIR); tar xf pigharness.tar)
+ (cd $(TEST_DIST_DIR); mv drivers/*.pm .)
+ mkdir -p $(TEST_DIST_DIR)/libexec/PigTest
+ (cd $(TEST_DIST_DIR); cp $(TOOLS) libexec/PigTest)
+ chmod +x $(TEST_DIST_DIR)/libexec/PigTest/*
+ cp $(CONF_DIR)/testpropertiesfile.conf $(TEST_DIST_DIR)/conf
+ mkdir -p $(TEST_DIST_DIR)/lib/java
+ cp $(UDF_JAR) $(TEST_DIST_DIR)/lib/java
+ mkdir -p $(TEST_DIST_DIR)/libexec/python
+ cp $(PYTHON_UDF_DIR)/*.py $(TEST_DIST_DIR)/libexec/python
+ mkdir -p $(TEST_DIST_DIR)/paramfiles
+ cp $(PARAM_FILES) $(TEST_DIST_DIR)/paramfiles
+ cp $(LIB_DIR)/* $(TEST_DIST_DIR)/lib
+ cp $(STREAMING_DIR)/* $(TEST_DIST_DIR)/libexec
+ chmod +x $(TEST_DIST_DIR)/libexec/*.pl
+ (cd $(TEST_DIST_DIR); PIG_HARNESS_ROOT=. PH_LOCAL=. PH_OUT=. PH_ROOT=. \
+ ./pig_test_harness.pl $(TESTS_TO_RUN) $(TESTS))
+
+clean:
+ rm -f $(ARCHIVE)
+ rm -rf $(TEST_DIST_DIR)
+ rm -rf $(JAVA_UDF_DIR)/build
+ rm -rf $(JAVA_UDF_DIR)/testudf.jar
+
+
Added: pig/trunk/test/e2e/pig/conf/default.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/conf/default.conf?rev=1087340&view=auto
==============================================================================
--- pig/trunk/test/e2e/pig/conf/default.conf (added)
+++ pig/trunk/test/e2e/pig/conf/default.conf Thu Mar 31 15:31:33 2011
@@ -0,0 +1,70 @@
+############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+my $me = `whoami`;
+chomp $me;
+
+# The contents of this file can be rewritten to fit your installation.
+# Also, you can define the following environment variables and set things up as in the test setup
+# PH_ROOT Root directory where test harness is installed
+# PH_LOCAL Root directory for input and output for local mode tests
+# PH_OUT Root directory where output data will be stored (on local disk, not HDFS)
+# PH_CLUSTER Root directory for cluster being used
+# PH_PIG Root directory for Pig version being used
+
+$cfg = {
+ #HDFS
+ 'inpathbase' => '/user/pig/tests/data'
+ , 'outpathbase' => '/user/pig/out'
+
+ #LOCAL
+ , 'localinpathbase' => "$ENV{PH_LOCAL}/in"
+ , 'localoutpathbase' => "$ENV{PH_LOCAL}/out/log"
+ , 'localxmlpathbase' => "$ENV{PH_LOCAL}/out/xml"
+ , 'localpathbase' => "$ENV{PH_LOCAL}/out/pigtest/$me"
+
+ #TEST
+ , 'benchmarkPath' => "$ENV{PH_OUT}/benchmarks"
+ , 'scriptPath' => "$ENV{PH_ROOT}/libexec"
+ , 'tmpPath' => '/tmp/pigtest'
+ , 'jythonjar' => "$ENV{PH_JYTHON_JAR}"
+
+ #PIG
+ , 'testconfigpath' => "$ENV{PH_CLUSTER}/conf/"
+ , 'funcjarPath' => "$ENV{PH_ROOT}/lib/java"
+ , 'paramPath' => "$ENV{PH_ROOT}/paramfiles"
+ , 'testjar' => "$ENV{PH_PIG}/pig.jar"
+
+ #HADOOP
+ , 'hadoopHome' => "$ENV{PH_ROOT}/lib"
+
+
+ #DATABASE
+ , 'dbdb' => 'pigtest'
+ , 'dbuser' => 'pigtester'
+ , 'dbowner' => 'hadoopqa'
+ , 'dbgroup' => 'users'
+ , 'dbroot' => '/var/lib/pgsql'
+ , 'dbdata' => '${db.root}/data '
+ , 'dblog' => '/var/log/pgsql'
+
+ , 'userhomePath' => "$ENV{HOME}"
+ ,'local.bin' => '/usr/bin'
+
+ ,'logDir' => "$ENV{PH_OUT}/log"
+ ,'propertiesFile' => "./conf/testpropertiesfile.conf"
+
+};
Added: pig/trunk/test/e2e/pig/conf/testpropertiesfile.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/conf/testpropertiesfile.conf?rev=1087340&view=auto
==============================================================================
--- pig/trunk/test/e2e/pig/conf/testpropertiesfile.conf (added)
+++ pig/trunk/test/e2e/pig/conf/testpropertiesfile.conf Thu Mar 31 15:31:33 2011
@@ -0,0 +1,21 @@
+############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+$cfg = {
+ 'pigharness.log' => './out/harness.log'
+ ,'pigharness.log.level' => 'DEBUG'
+ ,'pigharness.console.level'=> 'INFO'
+};
Added: pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm?rev=1087340&view=auto
==============================================================================
--- pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm (added)
+++ pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm Thu Mar 31 15:31:33 2011
@@ -0,0 +1,1146 @@
+package TestDriverPig;
+
+############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+###############################################################################
+# Test driver for pig nightly tests.
+#
+#
+
+use TestDriver;
+use IPC::Run; # don't do qw(run), it screws up TestDriver which also has a run method
+use Digest::MD5 qw(md5_hex);
+use Util;
+use File::Path;
+
+use strict;
+use English;
+
+our $className= "TestDriver";
+our @ISA = "$className";
+our $ROOT = (defined $ENV{'PIG_HARNESS_ROOT'} ? $ENV{'PIG_HARNESS_ROOT'} : die "ERROR: You must set environment variable PIG_HARNESS_ROOT\n");
+our $toolpath = "$ROOT/libexec/PigTest";
+
+my $passedStr = 'passed';
+my $failedStr = 'failed';
+my $abortedStr = 'aborted';
+my $skippedStr = 'skipped';
+my $dependStr = 'failed_dependency';
+
+sub new
+{
+ # Call our parent
+ my ($proto) = @_;
+ my $class = ref($proto) || $proto;
+ my $self = $class->SUPER::new;
+
+ $self->{'exectype'} = "mapred"; # till we know better (in globalSetup())!
+ $self->{'ignore'} = "true"; # till we know better (in globalSetup())!
+
+ bless($self, $class);
+ return $self;
+}
+
+###############################################################################
+# This method has been copied over from TestDriver to make changes to
+# support skipping tests which do not match current execution mode
+# or which were marked as 'ignore'
+#
+#
+# Static function, can be used by test_harness.pl
+# Print the results so far, given the testStatuses hash.
+# @param testStatuses - reference to hash of test status results.
+# @param log - reference to file handle to print results to.
+# @param prefix - A title to prefix to the results
+# @returns nothing.
+#
+sub printResults
+{
+ my ($testStatuses, $log, $prefix) = @_;
+
+ my ($pass, $fail, $abort, $depend, $skipped) = (0, 0, 0, 0, 0);
+
+ foreach (keys(%$testStatuses)) {
+ ($testStatuses->{$_} eq $passedStr) && $pass++;
+ ($testStatuses->{$_} eq $failedStr) && $fail++;
+ ($testStatuses->{$_} eq $abortedStr) && $abort++;
+ ($testStatuses->{$_} eq $dependStr) && $depend++;
+ ($testStatuses->{$_} eq $skippedStr) && $skipped++;
+ }
+
+ my $msg = "$prefix, PASSED: $pass FAILED: $fail SKIPPED: $skipped ABORTED: $abort " .
+ "FAILED DEPENDENCY: $depend";
+ print $log "$msg\n";
+ print "$msg\r";
+}
+
+
+sub replaceParameters
+{
+##!!! Move this to Util.pm
+
+ my ($self, $cmd, $outfile, $testCmd, $log) = @_;
+
+ # $self
+ $cmd =~ s/:LATESTOUTPUTPATH:/$self->{'latestoutputpath'}/g;
+
+ # $outfile
+ $cmd =~ s/:OUTPATH:/$outfile/g;
+
+ # $ENV
+ $cmd =~ s/:PIGHARNESS:/$ENV{PIG_HARNESS_ROOT}/g;
+
+ # $testCmd
+ $cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g;
+ $cmd =~ s/:OUTPATH:/$outfile/g;
+ $cmd =~ s/:FUNCPATH:/$testCmd->{'funcjarPath'}/g;
+ $cmd =~ s/:RUNID:/$testCmd->{'UID'}/g;
+ $cmd =~ s/:USRHOMEPATH:/$testCmd->{'userhomePath'}/g;
+ $cmd =~ s/:HADOOPHOME:/$testCmd->{'hadoopHome'}/g;
+ $cmd =~ s/:SCRIPTHOMEPATH:/$testCmd->{'scriptPath'}/g;
+ $cmd =~ s/:DBUSER:/$testCmd->{'dbuser'}/g;
+ $cmd =~ s/:DBNAME:/$testCmd->{'dbdb'}/g;
+ $cmd =~ s/:LOCALINPATH:/$testCmd->{'localinpathbase'}/g;
+ $cmd =~ s/:LOCALOUTPATH:/$testCmd->{'localoutpathbase'}/g;
+ $cmd =~ s/:LOCALTESTPATH:/$testCmd->{'localpathbase'}/g;
+ $cmd =~ s/:BMPATH:/$testCmd->{'benchmarkPath'}/g;
+ $cmd =~ s/:TMP:/$testCmd->{'tmpPath'}/g;
+
+ if ( $testCmd->{'hadoopSecurity'} eq "secure" ) {
+ $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteSecureCluster'}/g;
+ } else {
+ $cmd =~ s/:REMOTECLUSTER:/$testCmd->{'remoteNotSecureCluster'}/g;
+ }
+
+ return $cmd;
+}
+
+sub globalSetup
+{
+ my ($self, $globalHash, $log) = @_;
+ my $subName = (caller(0))[3];
+
+
+ # Setup the output path
+ my $me = `whoami`;
+ chomp $me;
+ $globalHash->{'runid'} = $me . "." . time;
+
+ # if "-ignore false" was provided on the command line,
+ # it means do run tests even when marked as 'ignore'
+ if(defined($globalHash->{'ignore'}) && $globalHash->{'ignore'} eq 'false')
+ {
+ $self->{'ignore'} = 'false';
+ }
+
+ # if "-x local" was provided on the command line,
+ # it implies pig should be run in "local" mode -so
+ # change input and output paths
+ if(defined($globalHash->{'x'}) && $globalHash->{'x'} eq 'local')
+ {
+ $self->{'exectype'} = "local";
+ $globalHash->{'inpathbase'} = $globalHash->{'localinpathbase'};
+ $globalHash->{'outpathbase'} = $globalHash->{'localoutpathbase'};
+ }
+ $globalHash->{'outpath'} = $globalHash->{'outpathbase'} . "/" . $globalHash->{'runid'} . "/";
+ $globalHash->{'localpath'} = $globalHash->{'localpathbase'} . "/" . $globalHash->{'runid'} . "/";
+
+ # extract the current zebra.jar file path from the classpath
+ # and enter it in the hash for use in the substitution of :ZEBRAJAR:
+ my $zebrajar = $globalHash->{'cp'};
+ $zebrajar =~ s/zebra.jar.*/zebra.jar/;
+ $zebrajar =~ s/.*://;
+ $globalHash->{'zebrajar'} = $zebrajar;
+
+ # add libexec location to the path
+ if (defined($ENV{'PATH'})) {
+ $ENV{'PATH'} = $globalHash->{'scriptPath'} . ":" . $ENV{'PATH'};
+ }
+ else {
+ $ENV{'PATH'} = $globalHash->{'scriptPath'};
+ }
+
+ my @cmd = (Util::getBasePigCmd($globalHash), '-e', 'mkdir', $globalHash->{'outpath'});
+
+ if($self->{'exectype'} eq "local")
+ {
+ @cmd = ('mkdir', '-p', $globalHash->{'outpath'});
+ }
+
+
+ print $log join(" ", @cmd);
+
+ if($self->{'exectype'} eq "mapred")
+ {
+ IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create HDFS directory " . $globalHash->{'outpath'} . ": $? - $!\n";
+ }
+ else
+ {
+ IPC::Run::run(\@cmd, \undef, $log, $log) or die "Cannot create directory " . $globalHash->{'outpath'} . "\n";
+ }
+
+ IPC::Run::run(['mkdir', '-p', $globalHash->{'localpath'}], \undef, $log, $log) or
+ die "Cannot create localpath directory " . $globalHash->{'localpath'} .
+ " " . "$ERRNO\n";
+}
+
+
+sub runTest
+{
+ my ($self, $testCmd, $log, $copyResults) = @_;
+ my $subName = (caller(0))[3];
+
+ # Handle the various methods of running used in
+ # the original TestDrivers
+
+ if ( $testCmd->{'pig'} && $self->hasCommandLineVerifications( $testCmd, $log) ) {
+ return $self->runPigCmdLine( $testCmd, $log, $copyResults );
+ } elsif( $testCmd->{'pig'} ){
+ return $self->runPig( $testCmd, $log, $copyResults );
+ #} elsif( $testCmd->{'pigsql'} ){
+ # return $self->runPigSql( $testCmd, $log, $copyResults );
+ } elsif( $testCmd->{'script'} ){
+ return $self->runScript( $testCmd, $log );
+ } else {
+ die "$subName FATAL Did not find a testCmd that I know how to handle";
+ }
+}
+
+
+sub runPigCmdLine
+{
+ my ($self, $testCmd, $log) = @_;
+ my $subName = (caller(0))[3];
+ my %result;
+
+ # Set up file locations
+ my $pigfile = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".pig";
+ my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
+
+ my $outdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
+ my $stdoutfile = "$outdir/stdout";
+ my $stderrfile = "$outdir/stderr";
+
+ mkpath( [ $outdir ] , 0, 0755) if ( ! -e outdir );
+ if ( ! -e $outdir ){
+ print $log "$0.$subName FATAL could not mkdir $outdir\n";
+ die "$0.$subName FATAL could not mkdir $outdir\n";
+ }
+
+ # Write the pig script to a file.
+ my $pigcmd = $self->replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log );
+
+ open(FH, "> $pigfile") or die "Unable to open file $pigfile to write pig script, $ERRNO\n";
+ print FH $pigcmd . "\n";
+ close(FH);
+
+ # Build the command
+ my @baseCmd = Util::getBasePigCmd($testCmd);
+ my @cmd = @baseCmd;
+
+ # Add option -l giving location for secondary logs
+ ##!!! Should that even be here?
+ my $locallog = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".log";
+ push(@cmd, "-logfile");
+ push(@cmd, $locallog);
+
+ # Add pig parameters if they're provided
+ if (defined($testCmd->{'pig_params'})) {
+ # Processing :PARAMPATH: in parameters
+ foreach my $param (@{$testCmd->{'pig_params'}}) {
+ $param =~ s/:PARAMPATH:/$testCmd->{'paramPath'}/g;
+ }
+ push(@cmd, @{$testCmd->{'pig_params'}});
+ }
+
+ # Add pig file and redirections
+ push(@cmd, $pigfile);
+ my $command= join (" ", @cmd) . " 1> $stdoutfile 2> $stderrfile";
+
+ # Run the command
+ print $log "$0:$subName Going to run command: ($command)\n";
+ print $log "$0:$subName STD OUT IS IN FILE ($stdoutfile)\n";
+ print $log "$0:$subName STD ERROR IS IN FILE ($stderrfile)\n";
+ print $log "$0:$subName PIG SCRIPT CONTAINS ($pigfile): \n$pigcmd\n";
+
+ my @result=`$command`;
+ $result{'rc'} = $? >> 8;
+ $result{'output'} = $outfile;
+ $result{'stdout'} = `cat $stdoutfile`;
+ $result{'stderr'} = `cat $stderrfile`;
+ $result{'stderr_file'} = $stderrfile;
+
+ print $log "STD ERROR CONTAINS:\n$result{'stderr'}\n";
+
+ return \%result;
+}
+
+
+sub runScript
+{
+ my ($self, $testCmd, $log) = @_;
+ my $subName = (caller(0))[3];
+ my %result;
+
+ # Set up file locations
+ my $script = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".sh";
+ my $outdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
+
+ my $outfile = "$outdir/script.out";
+ my $stdoutfile = "$outdir/script.out";
+ my $stderrfile = "$outdir/script.err";
+
+ mkpath( [ $outdir ] , 0, 0755) if ( ! -e outdir );
+ if ( ! -e $outdir ){
+ print $log "$0.$subName FATAL could not mkdir $outdir\n";
+ die "$0.$subName FATAL could not mkdir $outdir\n";
+ }
+
+ # Write the script to a file
+ my $cmd = $self->replaceParameters( $testCmd->{'script'}, $outfile, $testCmd, $log );
+
+ open(FH, ">$script") or die "Unable to open file $script to write script, $ERRNO\n";
+ print FH $cmd . "\n";
+ close(FH);
+
+ my @result=`chmod +x $script`;
+
+ # Build the command
+ my $command= "$script 1> $stdoutfile 2> $stderrfile";
+
+ # Run the script
+ print $log "$0:$subName Going to run command: ($command)\n";
+ print $log "$0:$subName STD OUT IS IN FILE ($stdoutfile)\n";
+ print $log "$0:$subName STD ERROR IS IN FILE ($stderrfile)\n";
+ print $log "$0:$subName SCRIPT CONTAINS ($script): \n$cmd\n";
+
+ @result=`$command`;
+ $result{'rc'} = $? >> 8;
+ $result{'output'} = $outfile;
+ $result{'stdout'} = `cat $stdoutfile`;
+ $result{'stderr'} = `cat $stderrfile`;
+ $result{'stderr_file'} = $stderrfile;
+
+ print $log "STD ERROR CONTAINS:\n$result{'stderr'}\n";
+
+ return \%result;
+}
+
+
+
+sub runPig
+{
+ my ($self, $testCmd, $log, $copyResults) = @_;
+ my $subName = (caller(0))[3];
+
+ my %result;
+
+ # Write the pig script to a file.
+ my $pigfile = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".pig";
+ my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
+
+ my $pigcmd = $self->replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log );
+
+ open(FH, "> $pigfile") or die "Unable to open file $pigfile to write pig script, $ERRNO\n";
+ print FH $pigcmd . "\n";
+ close(FH);
+
+
+ # Build the command
+ my @baseCmd = Util::getBasePigCmd($testCmd);
+ my @cmd = @baseCmd;
+
+ # Add option -l giving location for secondary logs
+ my $locallog = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".log";
+ push(@cmd, "-logfile");
+ push(@cmd, $locallog);
+
+ # Add pig parameters if they're provided
+ if (defined($testCmd->{'pig_params'})) {
+ # Processing :PARAMPATH: in parameters
+ foreach my $param (@{$testCmd->{'pig_params'}}) {
+ $param =~ s/:PARAMPATH:/$testCmd->{'paramPath'}/g;
+ }
+ push(@cmd, @{$testCmd->{'pig_params'}});
+ }
+
+ push(@cmd, $pigfile);
+
+
+ # Run the command
+ print $log "$0::$className::$subName INFO: Going to run pig command: @cmd\n";
+
+ IPC::Run::run(\@cmd, \undef, $log, $log) or
+ die "Failed running $pigfile\n";
+ $result{'rc'} = $? >> 8;
+
+
+ # Get results from the command locally
+ my $localoutfile;
+ my $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
+ my @SQLQuery = @{$testCmd->{'queries'}}; # here only used to determine if single-guery of multi-query
+
+ # mapreduce
+ if($self->{'exectype'} eq "mapred")
+ {
+ # single query
+ if ($#SQLQuery == 0) {
+ if ($copyResults) {
+ $result{'output'} = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log);
+ $result{'originalOutput'} = "$localdir/out_original"; # populated by postProcessSingleOutputFile
+ } else {
+ $result{'output'} = "NO_COPY";
+ }
+ }
+ # multi query
+ else {
+ my @outfiles = ();
+ for (my $id = 1; $id <= ($#SQLQuery + 1); $id++) {
+ $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out/$id";
+ $localoutfile = $outfile . ".$id";
+
+ # Copy result file out of hadoop
+ my $testOut;
+ if ($copyResults) {
+ $testOut = $self->postProcessSingleOutputFile($localoutfile, $localdir, \@baseCmd, $testCmd, $log);
+ } else {
+ $testOut = "NO_COPY";
+ }
+ push(@outfiles, $testOut);
+ }
+ ##!!! originalOutputs not set! Needed?
+ $result{'outputs'} = \@outfiles;
+ }
+ }
+ # local mode
+ else
+ {
+ # single query
+ if ($#SQLQuery == 0) {
+ $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".dir";
+ mkdir $localdir;
+ $result{'output'} = $self->postProcessSingleOutputFile($outfile, $localdir, \@baseCmd, $testCmd, $log);
+ $result{'originalOutput'} = "$localdir/out_original"; # populated by postProcessSingleOutputFile
+ }
+ # multi query
+ else {
+ my @outfiles = ();
+ for (my $id = 1; $id <= ($#SQLQuery + 1); $id++) {
+ $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
+ mkdir $localdir;
+ $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out/$id";
+ mkdir $localdir;
+ $localoutfile = $outfile . ".$id";
+
+ my $testRes = $self->postProcessSingleOutputFile($localoutfile, $localdir, \@baseCmd, $testCmd, $log);
+ push(@outfiles, $testRes);
+ }
+ ##!!! originalOutputs not set!
+ $result{'outputs'} = \@outfiles;
+ }
+ }
+
+ # Compare doesn't get the testCmd hash, so I need to stuff the necessary
+ # info about sorting into the result.
+ if (defined $testCmd->{'sortArgs'} && $testCmd->{'sortArgs'}) {
+ $result{'sortArgs'} = $testCmd->{'sortArgs'};
+ }
+
+ return \%result;
+}
+
+
+sub postProcessSingleOutputFile
+{
+ my ($self, $outfile, $localdir, $baseCmd, $testCmd, $log) = @_;
+ my $subName = (caller(0))[3];
+
+ # Copy to local if results on HDFS
+ if ($self->{'exectype'} eq "mapred") {
+ my @baseCmd = @{$baseCmd};
+ my @copyCmd = @baseCmd;
+ push(@copyCmd, ('-e', 'copyToLocal', $outfile, $localdir));
+ print $log "$0::$className::$subName INFO: Going to run pig command: @copyCmd\n";
+
+ IPC::Run::run(\@copyCmd, \undef, $log, $log) or die "Cannot copy results from HDFS $outfile to $localdir\n";
+ }
+
+
+ # Sort the result if necessary. Keep the original output in one large file.
+ # Use system not IPC run so that the '*' gets interpolated by the shell.
+
+ # Build command to:
+ # 1. Combine part files
+ my $fppCmd =
+ ($self->{'exectype'} eq "mapred") ? "cat $localdir/map* $localdir/part* 2>/dev/null" :
+ (-d $outfile ) ? "cat $outfile/part* 2>/dev/null" :
+ "cat $outfile";
+
+ # 2. Standardize float precision
+ if (defined $testCmd->{'floatpostprocess'} && defined $testCmd->{'delimiter'}) {
+ $fppCmd .= " | $toolpath/floatpostprocessor.pl '" . $testCmd->{'delimiter'} . "'";
+ }
+
+ $fppCmd .= " > $localdir/out_original";
+
+ # run command
+ print $log "$fppCmd\n";
+ system($fppCmd);
+
+ # Sort the results for the benchmark compare.
+ my @sortCmd = ('sort', "$localdir/out_original");
+ print $log join(" ", @sortCmd) . "\n";
+ IPC::Run::run(\@sortCmd, '>', "$localdir/out_sorted");
+
+ return "$localdir/out_sorted";
+}
+
+sub generateBenchmark
+{
+ my ($self, $testCmd, $log) = @_;
+
+ my %result;
+
+ my @SQLQuery = @{$testCmd->{'queries'}};
+
+ if ($#SQLQuery == 0) {
+ my $outfile = $self->generateSingleSQLBenchmark($testCmd, $SQLQuery[0], undef, $log);
+ $result{'output'} = $outfile;
+ } else {
+ my @outfiles = ();
+ for (my $id = 0; $id < ($#SQLQuery + 1); $id++) {
+ my $sql = $SQLQuery[$id];
+ my $outfile = $self->generateSingleSQLBenchmark($testCmd, $sql, ($id+1), $log);
+ push(@outfiles, $outfile);
+ }
+ $result{'outputs'} = \@outfiles;
+ }
+
+ return \%result;
+}
+
+sub generateSingleSQLBenchmark
+{
+ my ($self, $testCmd, $sql, $id, $log) = @_;
+
+ my $qmd5 = substr(md5_hex($testCmd->{'pig'}), 0, 5);
+ my $outfile = $testCmd->{'benchmarkPath'} . "/" . $testCmd->{'group'} . "_" . $testCmd->{'num'};
+ $outfile .= defined($id) ? ".$id" . ".expected." . $qmd5 : ".expected." . $qmd5;
+
+ if (-e $outfile) {
+ return $outfile;
+ }
+
+ my @cmd = ('psql', '-U', $testCmd->{'dbuser'}, '-d', $testCmd->{'dbdb'},
+ '-c', $sql, '-t', '-A', '--pset', "fieldsep= ", '-o', $outfile);
+
+ print $log "Running SQL command [" . join(" ", @cmd) . "\n";
+ IPC::Run::run(\@cmd, \undef, $log, $log) or do {
+ print $log "Sql command <" . $sql .
+ " failed for >>$testCmd->{group}_$testCmd->{num}<<\n";
+ unlink $outfile if ( -e $outfile );
+
+ die "Sql command failed for >>$testCmd->{group}_$testCmd->{num}<<\n";
+ };
+
+ # Sort and postprocess the result if necessary
+ my $sortCmd = "cat $outfile";
+ if (defined $testCmd->{'floatpostprocess'} && defined $testCmd->{'delimiter'}) {
+ $sortCmd .= " | $toolpath/floatpostprocessor.pl '" . $testCmd->{'delimiter'} . "' ";
+ }
+
+ # Use system not IPC run so that the '*' gets interpolated by the shell.
+ $sortCmd .= " | sort";
+ if (defined $testCmd->{'sortBenchmarkArgs'}) {
+ $sortCmd .= " " . join(" ", @{$testCmd->{'sortBenchmarkArgs'}});
+ }
+ my $tmpfile = $outfile . ".tmp";
+ $sortCmd .= " > $tmpfile";
+ print $log "$sortCmd\n";
+ system($sortCmd);
+ unlink $outfile;
+
+ IPC::Run::run ['mv', $tmpfile, $outfile];
+
+ return $outfile;
+}
+
+sub hasCommandLineVerifications
+{
+ my ($self, $testCmd, $log) = @_;
+
+ foreach my $key ('rc', 'expected_out', 'expected_out_regex', 'expected_err', 'expected_err_regex',
+ 'not_expected_out', 'not_expected_out_regex', 'not_expected_err', 'not_expected_err_regex' ) {
+ if (defined $testCmd->{$key}) {
+ return 1;
+ }
+ }
+ return 0;
+}
+
+
+sub compare
+{
+ my ($self, $testResult, $benchmarkResult, $log, $testCmd) = @_;
+ my $subName = (caller(0))[3];
+
+ # For now, if the test has
+ # - testCmd pig, and 'sql' for benchmark, then use comparePig, i.e. using benchmark
+ # - any verification directives formerly used by CmdLine or Script drivers (rc, regex on out and err...)
+ # then use compareScript even if testCmd is "pig"
+ # - testCmd script, then use compareScript
+ # - testCmd pig, and none of the above, then use comparePig
+ #
+ # (The first condition included because nightly.conf actually had a test, Types_2, that
+ # specified both 'sql' and 'expected_err_regex'.)
+ #
+ # Later, should add ability to have same tests both verify with the 'script' directives,
+ # and do a benchmark compare, if it was a pig cmd. E.g. 'rc' could still be checked when
+ # doing the benchmark compare.
+
+ if( defined $testCmd->{'sql'} ){
+ return $self->comparePig ( $testResult, $benchmarkResult, $log, $testCmd);
+ } elsif( $testCmd->{'script'} || $self->hasCommandLineVerifications( $testCmd, $log) ){
+ return $self->compareScript ( $testResult, $log, $testCmd);
+ } elsif( $testCmd->{'pig'} ){
+ return $self->comparePig ( $testResult, $benchmarkResult, $log, $testCmd);
+ #} elsif( $testCmd->{'pigsql'} ){
+ # return $self->comparePigSql ( $testResult, $benchmarkResult, $log, $testCmd);
+ } else {
+ # Should have been caught by runTest, still...
+ print $log "$0.$subName WARNING Did not find a testCmd that I know how to handle\n";
+ return 0;
+ }
+}
+
+
+sub compareScript
+{
+ my ($self, $testResult, $log, $testCmd) = @_;
+ my $subName = (caller(0))[3];
+
+ # IMPORTANT NOTES:
+ #
+ # If you are using a regex to compare stdout or stderr
+ # and if the pattern that you are trying to match spans two line
+ # explicitly use '\n' (without the single quotes) in the regex
+ #
+ # If any verification directives are added here
+ # do remember also to add them to the hasCommandLineVerifications subroutine.
+ #
+ # If the test conf file misspells the directive, you won't be told...
+ #
+
+ my $result = 1; # until proven wrong...
+
+
+ # Return Code
+ if (defined $testCmd->{'rc'}) {
+ print $log "$0::$subName INFO Checking return code " .
+ "against expected <$testCmd->{'rc'}>\n";
+ if ( (! defined $testResult->{'rc'}) || ($testResult->{'rc'} != $testCmd->{'rc'})) {
+ print $log "$0::$subName INFO Check failed: rc = <$testCmd->{'rc'}> expected, test returned rc = <$testResult->{'rc'}>\n";
+ $result = 0;
+ }
+ }
+
+ # Standard Out
+ if (defined $testCmd->{'expected_out'}) {
+ print $log "$0::$subName INFO Checking test stdout' " .
+ "as exact match against expected <$testCmd->{'expected_out'}>\n";
+ if ($testResult->{'stdout'} ne $testCmd->{'expected_out'}) {
+ print $log "$0::$subName INFO Check failed: exact match of <$testCmd->{'expected_out'}> expected in stdout: $testResult->{'output'}\n";
+ $result = 0;
+ }
+ }
+
+ if (defined $testCmd->{'not_expected_out'}) {
+ print $log "$0::$subName INFO Checking test stdout " .
+ "as NOT exact match against expected <$testCmd->{'expected_out'}>\n";
+ if ($testResult->{'stdout'} eq $testCmd->{'not_expected_out'}) {
+ print $log "$0::$subName INFO Check failed: NON-match of <$testCmd->{'expected_out'}> expected to stdout: $testResult->{'output'}\n";
+ $result = 0;
+ }
+ }
+
+ if (defined $testCmd->{'expected_out_regex'}) {
+ print $log "$0::$subName INFO Checking test stdout " .
+ "for regular expression <$testCmd->{'expected_out_regex'}>\n";
+ if ($testResult->{'stdout'} !~ $testCmd->{'expected_out_regex'}) {
+ print $log "$0::$subName INFO Check failed: regex match of <$testCmd->{'expected_out_regex'}> expected in stdout: $testResult->{'output'}\n";
+ $result = 0;
+ }
+ }
+
+ if (defined $testCmd->{'not_expected_out_regex'}) {
+ print $log "$0::$subName INFO Checking test stdout " .
+ "for NON-match of regular expression <$testCmd->{'not_expected_out_regex'}>\n";
+ if ($testResult->{'stdout'} =~ $testCmd->{'not_expected_out_regex'}) {
+ print $log "$0::$subName INFO Check failed: regex NON-match of <$testCmd->{'not_expected_out_regex'}> expected in stdout: $testResult->{'output'}\n";
+ $result = 0;
+ }
+ }
+
+ # Standard Error
+ if (defined $testCmd->{'expected_err'}) {
+ print $log "$0::$subName INFO Checking test stderr " .
+ "as exact match against expected <$testCmd->{'expected_err'}>\n";
+ if ($testResult->{'stderr'} ne $testCmd->{'expected_err'}) {
+ print $log "$0::$subName INFO Check failed: exact match of <$testCmd->{'expected_err'}> expected in stderr: $testResult->{'stderr_file'}\n";
+ $result = 0;
+ }
+ }
+
+ if (defined $testCmd->{'not_expected_err'}) {
+ print $log "$0::$subName INFO Checking test stderr " .
+ "as NOT an exact match against expected <$testCmd->{'expected_err'}>\n";
+ if ($testResult->{'stderr'} eq $testCmd->{'not_expected_err'}) {
+ print $log "$0::$subName INFO Check failed: NON-match of <$testCmd->{'expected_err'}> expected to stderr: $testResult->{'stderr_file'}\n";
+ $result = 0;
+ }
+ }
+
+ if (defined $testCmd->{'expected_err_regex'}) {
+ print $log "$0::$subName INFO Checking test stderr " .
+ "for regular expression <$testCmd->{'expected_err_regex'}>\n";
+ if ($testResult->{'stderr'} !~ $testCmd->{'expected_err_regex'}) {
+ print $log "$0::$subName INFO Check failed: regex match of <$testCmd->{'expected_err_regex'}> expected in stderr: $testResult->{'stderr_file'}\n";
+ $result = 0;
+ }
+ }
+
+ if (defined $testCmd->{'not_expected_err_regex'}) {
+ print $log "$0::$subName INFO Checking test stderr " .
+ "for NON-match of regular expression <$testCmd->{'not_expected_err_regex'}>\n";
+ if ($testResult->{'stderr'} =~ $testCmd->{'not_expected_err_regex'}) {
+ print $log "$0::$subName INFO Check failed: regex NON-match of <$testCmd->{'not_expected_err_regex'}> expected in stderr: $testResult->{'stderr_file'}\n";
+ $result = 0;
+ }
+ }
+
+ return $result;
+}
+
+
+# sub comparePigSql
+# {
+# my ($self, $testResult, $benchmarkResult, $testCmd, $log) = @_;
+# my $subName = (caller(0))[3];
+#
+# my $result;
+# ##!!! STUB
+# }
+
+
+sub comparePig
+{
+ my ($self, $testResult, $benchmarkResult, $log, $testCmd) = @_;
+ my $subName = (caller(0))[3];
+
+ my $result;
+ my @SQLQuery = @{$testCmd->{'queries'}};
+
+ if ($#SQLQuery == 0) {
+ $result = $self->compareSingleOutput($testResult, $testResult->{'output'},
+ $benchmarkResult->{'output'}, $log);
+ } else {
+ my $res = 0;
+ for (my $id = 0; $id < ($#SQLQuery + 1); $id++) {
+ my $testOutput = ($testResult->{'outputs'})->[$id];
+ my $benchmarkOutput = ($benchmarkResult->{'outputs'})->[$id];
+ $res += $self->compareSingleOutput($testResult, $testOutput,
+ $benchmarkOutput, $log);
+ $result = ($res == ($#SQLQuery + 1)) ? 1 : 0;
+ }
+ }
+
+ return $result;
+}
+
+
+sub compareSingleOutput
+{
+ my ($self, $testResult, $testOutput, $benchmarkOutput, $log) = @_;
+
+ # cksum the the two files to see if they are the same
+ my ($testChksm, $benchmarkChksm);
+ IPC::Run::run((['cat', $testOutput], '|', ['cksum']), \$testChksm,
+ $log) or die "$0: error: cannot run cksum on test results\n";
+ IPC::Run::run((['cat', $benchmarkOutput], '|', ['cksum']),
+ \$benchmarkChksm, $log) or die "$0: error: cannot run cksum on benchmark\n";
+
+ chomp $testChksm;
+ chomp $benchmarkChksm;
+ print $log "test cksum: $testChksm\nbenchmark cksum: $benchmarkChksm\n";
+
+ my $result;
+ if ($testChksm ne $benchmarkChksm) {
+ print $log "Test output checksum does not match benchmark checksum\n";
+ print $log "Test checksum = <$testChksm>\n";
+ print $log "Expected checksum = <$benchmarkChksm>\n";
+ print $log "RESULTS DIFFER: vimdiff $testOutput $benchmarkOutput\n";
+ } else {
+ $result = 1;
+ }
+
+ # Now, check if the sort order is specified
+ if (defined($testResult->{'sortArgs'})) {
+ Util::setLocale();
+ my @sortChk = ('sort', '-cs');
+ push(@sortChk, @{$testResult->{'sortArgs'}});
+ push(@sortChk, $testResult->{'originalOutput'});
+ print $log "Going to run sort check command: " . join(" ", @sortChk) . "\n";
+ IPC::Run::run(\@sortChk, \undef, $log, $log);
+ my $sortrc = $?;
+ if ($sortrc) {
+ print $log "Sort check failed\n";
+ $result = 0;
+ }
+ }
+
+ return $result;
+}
+
+###############################################################################
+# This method has been copied over from TestDriver to make changes to
+# support skipping tests which do not match current execution mode
+#
+#
+# Run all the tests in the configuration file.
+# @param testsToRun - reference to array of test groups and ids to run
+# @param testsToMatch - reference to array of test groups and ids to match.
+# If a test group_num matches any of these regular expressions it will be run.
+# @param cfg - reference to contents of cfg file
+# @param log - reference to a stream pointer for the logs
+# @param dbh - reference database connection
+# @param testStatuses- reference to hash of test statuses
+# @param confFile - config file name
+# @param startat - test to start at.
+# @returns nothing
+# failed.
+#
+sub run
+{
+ my ($self, $testsToRun, $testsToMatch, $cfg, $log, $dbh, $testStatuses,
+ $confFile, $startat, $logname ) = @_;
+ my $subName = (caller(0))[3];
+
+ my $msg="";
+ my $duration=0;
+ my $totalDuration=0;
+ my $groupDuration=0;
+
+
+ my $sawstart = !(defined $startat);
+ # Rather than make each driver handle our multi-level cfg, we'll flatten
+ # the hashes into one for it.
+ my %globalHash;
+
+ my $runAll = ((scalar(@$testsToRun) == 0) && (scalar(@$testsToMatch) == 0));
+
+ # Read the global keys
+ foreach (keys(%$cfg)) {
+ next if $_ eq 'groups';
+ $globalHash{$_} = $cfg->{$_};
+ }
+
+ # Do the global setup
+ $self->globalSetup(\%globalHash, $log);
+
+ my $report=0;
+ my $properties= new Properties( 0, $globalHash{'propertiesFile'} );
+
+ # For the xml directory, use the default directory from the configuration file
+ # unless the directory was specified in the command line
+
+ my $xmlDir= $globalHash{'localxmlpathbase'} ."/run". $globalHash{'UID'};
+ if ( $globalHash{'reportdir'} ) {
+ $xmlDir = $globalHash{'reportdir'};
+ }
+
+ my %groupExecuted;
+
+ foreach my $group (@{$cfg->{'groups'}}) {
+ my %groupHash = %globalHash;
+ $groupHash{'group'} = $group->{'name'};
+
+ # Read the group keys
+ foreach (keys(%$group)) {
+ next if $_ eq 'tests';
+ $groupHash{$_} = $group->{$_};
+ }
+
+ # Run each test
+ foreach my $test (@{$group->{'tests'}}) {
+ # Check if we're supposed to run this one or not.
+ if (!$runAll) {
+ # check if we are supposed to run this test or not.
+ my $foundIt = 0;
+ foreach (@$testsToRun) {
+ if (/^$groupHash{'group'}(_[0-9]+)?$/) {
+ if (not defined $1) {
+ # In this case it's just the group name, so we'll
+ # run every test in the group
+ $foundIt = 1;
+ last;
+ } else {
+ # maybe, it at least matches the group
+ my $num = "_" . $test->{'num'};
+ if ($num eq $1) {
+ $foundIt = 1;
+ last;
+ }
+ }
+ }
+ }
+ foreach (@$testsToMatch) {
+ my $protoName = $groupHash{'group'} . "_" . $test->{'num'};
+ if ($protoName =~ /$_/) {
+ if (not defined $1) {
+ # In this case it's just the group name, so we'll
+ # run every test in the group
+ $foundIt = 1;
+ last;
+ } else {
+ # maybe, it at least matches the group
+ my $num = "_" . $test->{'num'};
+ if ($num eq $1) {
+ $foundIt = 1;
+ last;
+ }
+ }
+ }
+ }
+
+ next unless $foundIt;
+ }
+
+ # This is a test, so run it.
+ my %testHash = %groupHash;
+ foreach (keys(%$test)) {
+ $testHash{$_} = $test->{$_};
+ }
+ my $testName = $testHash{'group'} . "_" . $testHash{'num'};
+
+ if ( $groupExecuted{ $group->{'name'} }== 0 ){
+ $groupExecuted{ $group->{'name'} }=1;
+
+ mkpath( [ $xmlDir ] , 1, 0777) if ( ! -e $xmlDir );
+
+ my $filename = $group->{'name'}.".xml";
+ $report = new TestReport ( $properties, "$xmlDir/$filename" );
+ $report->purge();
+
+ }
+
+ # Have we not reached the starting point yet?
+ if (!$sawstart) {
+ if ($testName eq $startat) {
+ $sawstart = 1;
+ } else {
+ next;
+ }
+ }
+
+ # Check that this test doesn't depend on an earlier test or tests
+ # that failed, or that the test wasn't marked as "ignore".
+ # Don't abort if that test wasn't run, just assume the
+ # user knew what they were doing and set it up right.
+ my $skipThisOne = 0;
+ foreach (keys(%testHash)) {
+ if (/^depends_on/ && defined($testStatuses->{$testHash{$_}}) &&
+ $testStatuses->{$testHash{$_}} ne $passedStr) {
+ print $log "Skipping test $testName, it depended on " .
+ "$testHash{$_} which returned a status of " .
+ "$testStatuses->{$testHash{$_}}\n";
+ $testStatuses->{$testName} = $dependStr;
+ $skipThisOne = 1;
+ last;
+ }
+ # if the test is not applicable to current execution mode
+ # ignore it
+ if(/^exectype$/i && $testHash{$_} !~ /$self->{'exectype'}/i)
+ {
+ print $log "\nIGNORING test since running mode ($self->{'exectype'}) and exectype in test ($testHash{'exectype'}) do not match\n";
+ $testStatuses->{$testName} = $skippedStr;
+ $skipThisOne = 1;
+ last;
+ }
+
+ # if the test is marked as 'ignore',
+ # ignore it... unless option to ignore the ignore is in force
+ if(/^ignore$/i && ($self->{'ignore'} eq 'true'))
+ {
+ print $log "$0::$className::$subName TEST IGNORED <$testName> at " . time . ". Message: $testHash{'ignore'}\n";
+ $testStatuses->{$testName} = $skippedStr;
+ $skipThisOne = 1;
+ last;
+ }
+ }
+
+ if ($skipThisOne) {
+ printResults($testStatuses, $log, "Results so far");
+ next;
+ }
+
+ # Check if output comparison should be skipped.
+ my $dontCompareThisOne = 0; # true for tests with key 'noverify'
+ my $copyResults = 1; # no need to copy output to local if noverify
+ foreach (keys(%testHash)) {
+
+ if(/^noverify$/i )
+ {
+ $dontCompareThisOne = 1;
+ $copyResults = 0;
+ last;
+ }
+ }
+
+ print $log "Beginning test $testName at " . time . "\n";
+ my %dbinfo = (
+ 'testrun_id' => $testHash{'trid'},
+ 'test_type' => $testHash{'driver'},
+ #'test_file' => $testHash{'file'},
+ 'test_file' => $confFile,
+ 'test_group' => $testHash{'group'},
+ 'test_num' => $testHash{'num'},
+ );
+ my $beginTime = time;
+ my ($testResult, $benchmarkResult);
+ my $msg;
+ eval {
+
+
+ my @SQLQuery = split /;/, $testHash{'sql'};
+
+ # Throw out the last one if it is just space
+ if ($SQLQuery[$#SQLQuery] =~ /^\s*$/) { $#SQLQuery--; }
+
+ # If the last one is a comment, decrement the count
+ if ($#SQLQuery > 0 && $SQLQuery[$#SQLQuery] !~ /select/i && $SQLQuery[$#SQLQuery] =~ /--/) {
+ $#SQLQuery--;
+ }
+
+ $testHash{'queries'} = \@SQLQuery;
+
+ $testResult = $self->runTest(\%testHash, $log, $copyResults);
+ my $endTime = time;
+
+ $benchmarkResult = $self->generateBenchmark(\%testHash, $log);
+
+ my $result;
+ if( $dontCompareThisOne ) {
+ $result = 1;
+ print $log "$0::$className::$subName TEST MARKED NOVERIFY <$testName>\n";
+ } else {
+ $result = $self->compare($testResult, $benchmarkResult, $log, \%testHash);
+ print $log "Test $testName\n";
+ }
+
+ if ($result) {
+ $msg = "TEST SUCCEEDED <$testName> at " . time . "\n";
+ $testStatuses->{$testName} = $passedStr;
+ } else {
+ $msg= "TEST FAILED <$testName> at " . time . "\n";
+ $testStatuses->{$testName} = $failedStr;
+ }
+ print $log $msg;
+
+ $dbinfo{'duration'} = $endTime - $beginTime;
+ $self->recordResults($result, $testResult, $benchmarkResult,
+ \%dbinfo, $log);
+ };
+
+ if ($@) {
+ my $endTime = time;
+ print $log "$0::$subName FAILED: Failed to run test $testName <$@>\n";
+ $testStatuses->{$testName} = $abortedStr;
+ $dbinfo{'duration'} = $beginTime - $endTime;
+ }
+
+ eval {
+ $dbinfo{'status'} = $testStatuses->{$testName};
+ if($dbh) {
+ $dbh->insertTestCase(\%dbinfo);
+ }
+ };
+ if ($@) {
+ chomp $@;
+ warn "Failed to insert test case info, error <$@>\n";
+ }
+
+ $self->cleanup($testStatuses->{$testName}, \%testHash, $testResult,
+ $benchmarkResult, $log);
+
+ #$report->testcase( $group->{'name'}, $testName, $duration, $msg, $testStatuses->{$testName}, $testResult ) if ( $report );
+ $report->testcase( $group->{'name'}, $testName, $duration, $msg, $testStatuses->{$testName} ) if ( $report );
+ $groupDuration = $groupDuration + $duration;
+ $totalDuration = $totalDuration + $duration;
+ printResults($testStatuses, $log, "Results so far");
+ }
+ if ( $report ) {
+ my $reportname= $group->{'name'};
+ if ( $globalHash{'reportname'} ) {
+ $reportname= $globalHash{'reportname'};
+ }
+ $report->systemOut( $logname, $reportname );
+ printGroupResultsXml( $report, $reportname, $testStatuses, $groupDuration );
+ }
+ $report = 0;
+ $groupDuration=0;
+
+ }
+
+ # Do the global cleanup
+ $self->globalCleanup(\%globalHash, $log);
+}
+
+##############################################################################
+# Sub: printGroupResultsXml
+# Print the results for the group using junit xml schema using values from the testStatuses hash.
+#
+# Paramaters:
+# $report - the report object to use to generate the report
+# $groupName - the name of the group to report totals for
+# $testStatuses - the hash containing the results for the tests run so far
+# $totalDuration- The total time it took to run the group of tests
+#
+# Returns:
+# None.
+#
+sub printGroupResultsXml
+{
+ my ( $report, $groupName, $testStatuses, $totalDuration) = @_;
+ $totalDuration=0 if ( !$totalDuration );
+
+ my ($pass, $fail, $abort, $depend) = (0, 0, 0, 0);
+
+ foreach my $key (keys(%$testStatuses)) {
+ if ( $key =~ /^$groupName/ ){
+ ($testStatuses->{$key} eq $passedStr) && $pass++;
+ ($testStatuses->{$key} eq $failedStr) && $fail++;
+ ($testStatuses->{$key} eq $abortedStr) && $abort++;
+ ($testStatuses->{$key} eq $dependStr) && $depend++;
+ }
+ }
+
+ my $total= $pass + $fail + $abort;
+ $report->totals( $groupName, $total, $fail, $abort, $totalDuration );
+
+}
+
+1;
Added: pig/trunk/test/e2e/pig/drivers/TestDriverScript.pm
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/drivers/TestDriverScript.pm?rev=1087340&view=auto
==============================================================================
--- pig/trunk/test/e2e/pig/drivers/TestDriverScript.pm (added)
+++ pig/trunk/test/e2e/pig/drivers/TestDriverScript.pm Thu Mar 31 15:31:33 2011
@@ -0,0 +1,429 @@
+############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package TestDriverScript;
+
+###############################################################################
+# Test driver for pig nightly tests.
+#
+#
+my $ROOT=undef;
+if (defined $ENV{'PIG_HARNESS_ROOT'} ){
+ $ROOT= $ENV{'PIG_HARNESS_ROOT'};
+
+} else {
+ die "FATAL ERROR: $0 - You must set PIG_HARNESS_ROOT to the root directory of the pig_harness";
+}
+
+#Set library paths
+unshift( @INC, "$ROOT/libexec" );
+unshift( @INC, ".");
+
+use TestDriverPig;
+use IPC::Run; # don't do qw(run), it screws up TestDriver which also has a run method
+use File::Path;
+use Digest::MD5 qw(md5_hex);
+use Util;
+
+use strict;
+use English;
+
+our @ISA = "TestDriverPig";
+
+#########################################################################
+# Sub: new
+# TestDriverPigCmdline Constructor.
+#
+#
+# Paramaters:
+# None
+#
+# Returns:
+# None
+
+sub new
+{
+ # Call our parent
+ my ($proto) = @_;
+ my $class = ref($proto) || $proto;
+ my $self = $class->SUPER::new;
+ $self->{'ignore'} = "true";
+
+ bless($self, $class);
+ return $self;
+
+}
+
+########################################################################
+# Sub: runTest
+# Runs the test and returns the results.
+# -Write the pig script to a file.
+# -Run the command
+# -Copy result file out of hadoop
+# -Sort and postprocess the result if necessary
+#
+# Parameters:
+# $testCmd -
+# $log -
+#
+# Returns:
+# hash reference containg the test result
+#
+
+
+sub runTest
+{
+ my ($self, $testCmd, $log) = @_;
+ my $subName = (caller(0))[3];
+
+ my %result;
+
+ # extract the current zebra.jar file path from the classpath
+ # and enter it in the hash for use in the substitution of :ZEBRAJAR:
+ my $zebrajar = $testCmd->{'cp'};
+ $zebrajar =~ s/zebra.jar.*/zebra.jar/;
+ $zebrajar =~ s/.*://;
+ $testCmd->{'zebrajar'} = $zebrajar;
+
+ if( $testCmd->{'pig'} ){
+ return runPig( $self, $testCmd, $log );
+ } elsif( $testCmd->{'pigsql'} ){
+ return runPigSql( $self, $testCmd, $log );
+ } elsif( $testCmd->{'script'} ){
+ return runScript( $self, $testCmd, $log );
+ }
+
+ return %result;
+}
+
+sub runPig
+{
+ my ($self, $testCmd, $log) = @_;
+ my $subName = (caller(0))[3];
+ my %result;
+
+ return 1 if ( $testCmd->{'ignore'}=~ "true" );
+ # Write the pig script to a file.
+ my $pigfile = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".pig";
+ my $outdir = $testCmd->{'outlpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
+ my $outfile = "$outdir/pig.out";
+
+ my $pigcmd = $testCmd->{'pig'};
+ $pigcmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g;
+ $pigcmd =~ s/:OUTPATH:/$outfile/g;
+ $pigcmd =~ s/:FUNCPATH:/$testCmd->{'funcjarPath'}/g;
+ $pigcmd =~ s/:ZEBRAJAR:/$testCmd->{'zebrajar'}/g;
+ $pigcmd =~ s/:RUNID:/$testCmd->{'UID'}/g;
+ $pigcmd =~ s/:PIGHARNESS:/$ENV{PIG_HARNESS_ROOT}/g;
+ $pigcmd =~ s/:USRHOMEPATH:/$testCmd->{'userhomePath'}/g;
+ $pigcmd =~ s/:SCRIPTHOMEPATH:/$testCmd->{'scriptPath'}/g;
+ $pigcmd =~ s/:DBUSER:/$testCmd->{'dbuser'}/g;
+ $pigcmd =~ s/:DBNAME:/$testCmd->{'dbdb'}/g;
+ $pigcmd =~ s/:LOCALINPATH:/$testCmd->{'localinpathbase'}/g;
+ $pigcmd =~ s/:LOCALOUTPATH:/$testCmd->{'localoutpathbase'}/g;
+ $pigcmd =~ s/:BMPATH:/$testCmd->{'benchmarkPath'}/g;
+ $pigcmd =~ s/:TMP:/$testCmd->{'tmpPath'}/g;
+ $pigcmd =~ s/:FILER:/$testCmd->{'filerPath'}/g;
+
+ $pigcmd =~ s/:LATESTOUTPUTPATH:/$self->{'latestoutputpath'}/g;
+
+ #my $pigcmd = replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log );
+
+ open(FH, ">$pigfile") or die "Unable to open file $pigfile to write pig script, $ERRNO\n";
+ print FH $pigcmd . "\n";
+ close(FH);
+
+ # Run the command
+ my @cmd = Util::getBasePigCmd($testCmd);
+
+ # Add option -l giving location for secondary logs
+ my $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
+ mkdir $localdir;
+ my $locallog = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".log";
+ push(@cmd, "-l");
+ push(@cmd, $locallog);
+
+ # Add pig parameters if they're provided
+ if (defined($testCmd->{'pig_params'})) {
+ push(@cmd, @{$testCmd->{'pig_params'}});
+ }
+
+ push(@cmd, $pigfile);
+
+ print $log "Going to run pig command: @cmd\n";
+ print $log "Pig script contains: $pigcmd\n";
+
+ IPC::Run::run(\@cmd, \undef, \$result{'stdout'}, \$result{'stderr'});
+# print `@cmd`;
+ $result{'rc'} = $? >> 8;
+ return \%result;
+}
+
+
+sub runPigSql
+{
+ my ($self, $testCmd, $log) = @_;
+ my $subName = (caller(0))[3];
+ my %result;
+
+ return 1 if ( $testCmd->{'ignore'}=~ "true" );
+ # Write the pig script to a file.
+ my $pigfile = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".pig";
+ my $outdir = $testCmd->{'outlpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
+ my $outfile = "$outdir/pig.out";
+
+ my $pigcmd = $testCmd->{'pigsql'};
+ $pigcmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g;
+ $pigcmd =~ s/:OUTPATH:/$outfile/g;
+ $pigcmd =~ s/:FUNCPATH:/$testCmd->{'funcjarPath'}/g;
+ $pigcmd =~ s/:RUNID:/$testCmd->{'UID'}/g;
+ $pigcmd =~ s/:PIGHARNESS:/$ENV{PIG_HARNESS_ROOT}/g;
+ $pigcmd =~ s/:USRHOMEPATH:/$testCmd->{'userhomePath'}/g;
+ $pigcmd =~ s/:SCRIPTHOMEPATH:/$testCmd->{'scriptPath'}/g;
+ $pigcmd =~ s/:DBUSER:/$testCmd->{'dbuser'}/g;
+ $pigcmd =~ s/:DBNAME:/$testCmd->{'dbdb'}/g;
+ $pigcmd =~ s/:LOCALINPATH:/$testCmd->{'localinpathbase'}/g;
+ $pigcmd =~ s/:LOCALOUTPATH:/$testCmd->{'localoutpathbase'}/g;
+ $pigcmd =~ s/:BMPATH:/$testCmd->{'benchmarkPath'}/g;
+ $pigcmd =~ s/:TMP:/$testCmd->{'tmpPath'}/g;
+ $pigcmd =~ s/:FILER:/$testCmd->{'filerPath'}/g;
+
+ $pigcmd =~ s/:LATESTOUTPUTPATH:/$self->{'latestoutputpath'}/g;
+
+ #my $pigcmd = replaceParameters( $testCmd->{'pig'}, $outfile, $testCmd, $log );
+
+ open(FH, ">$pigfile") or die "Unable to open file $pigfile to write pig script, $ERRNO\n";
+ print FH $pigcmd . "\n";
+ close(FH);
+
+ # Run the command
+ my $outfile = $testCmd->{'outpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
+ $testCmd->{'testoutpath'}=$outfile;
+ $self->{'latestoutputpath'}=$outfile;
+
+ # Pig _SQL_ command
+ my @cmd = Util::getBasePigSqlCmd($testCmd);
+
+ # Add option -l giving location for secondary logs
+ my $localdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
+ mkdir $localdir;
+ my $locallog = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".log";
+ push(@cmd, "-l");
+ push(@cmd, $locallog);
+
+ # Add pig parameters if they're provided
+ if (defined($testCmd->{'pig_params'})) {
+ push(@cmd, @{$testCmd->{'pig_params'}});
+ }
+
+ push(@cmd, $pigfile);
+ print $log "Going to run pig sql command: @cmd\n";
+ print $log "Pig Sql script contains: $pigcmd\n";
+ IPC::Run::run(\@cmd, \undef, \$result{'stdout'}, \$result{'stderr'});
+ $result{'rc'} = $? >> 8;
+ return \%result;
+}
+
+
+########################################################################
+# Sub: runScript
+# Runs the Script and returns the results.
+# -Write the script to a file.
+# -Run the command
+#
+# Parameters:
+# $testCmd -
+# $log -
+#
+# Returns:
+# hash reference containg the test result
+#
+
+sub runScript
+{
+ my ($self, $testCmd, $log) = @_;
+ my $subName = (caller(0))[3];
+ my %result;
+
+ return 1 if ( $testCmd->{'ignore'}=~ "true" );
+
+ # Write the pig script to a file.
+ my $script = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".sh";
+
+ my $outdir = $testCmd->{'localpath'} . $testCmd->{'group'} . "_" . $testCmd->{'num'} . ".out";
+ print $log "Attempting to create $outdir\n";
+ mkpath( [ $outdir ] , 1, 0755) if ( ! -e outdir );
+ if ( ! -e $outdir ){
+ print $log "$0.$subName FATAL could not mkdir $outdir\n";
+ die "$0.$subName FATAL could not mkdir $outdir\n";
+ }
+ my $outfile = "$outdir/script.out";
+
+ my $cmd = $testCmd->{'script'};
+ $cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g;
+ $cmd =~ s/:OUTPATH:/$outfile/g;
+ $cmd =~ s/:FUNCPATH:/$testCmd->{'funcjarPath'}/g;
+ $cmd =~ s/:RUNID:/$testCmd->{'UID'}/g;
+ $cmd =~ s/:PIGHARNESS:/$ENV{PIG_HARNESS_ROOT}/g;
+ $cmd =~ s/:USRHOMEPATH:/$testCmd->{'userhomePath'}/g;
+ $cmd =~ s/:SCRIPTHOMEPATH:/$testCmd->{'scriptPath'}/g;
+ $cmd =~ s/:DBUSER:/$testCmd->{'dbuser'}/g;
+ $cmd =~ s/:DBNAME:/$testCmd->{'dbdb'}/g;
+ $cmd =~ s/:LOCALINPATH:/$testCmd->{'localinpathbase'}/g;
+ $cmd =~ s/:LOCALOUTPATH:/$testCmd->{'localoutpathbase'}/g;
+ $cmd =~ s/:BMPATH:/$testCmd->{'benchmarkPath'}/g;
+ $cmd =~ s/:TMP:/$testCmd->{'tmpPath'}/g;
+ $cmd =~ s/:FILER:/$testCmd->{'filerPath'}/g;
+# my $cmd = replaceParameters( $testCmd->{'script'}, $outfile, $testCmd, $log );
+
+ open(FH, ">$script") or die "Unable to open file $script to write pig script, $ERRNO\n";
+ print FH $cmd . "\n";
+ close(FH);
+
+ #my @cmds = split (/$/, $cmd);
+ #push(@cmds, $cmd);
+ print $log "$0:$subName RESULT ARE IN FILE ($outfile)\n";
+ print $log "$0:$subName SCRIPT CONTAINS ($script): \n$cmd\n";
+ my @result=`chmod +x $script`;
+ my $command= "$script >> $outfile 2>&1";
+ print $log "Going to run command: ($command)\n";
+ @result=`$command`;
+ $result{'rc'} = $?;
+ print $log @result;
+# IPC::Run::run(\@cmds, \undef, \$result{'stdout'}, \$result{'stderr'});
+ #IPC::Run::run(\@cmds, \undef, \$result{'stdout'}, \$result{'stderr'});
+ #$result{'rc'} = $? >> 8;
+ return \%result;
+}
+
+
+sub replaceParameters(){
+ my ($self, $cmd, $outfile, $testCmd, $log) = @_;
+
+ $cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g;
+ $cmd =~ s/:OUTPATH:/$outfile/g;
+ $cmd =~ s/:FUNCPATH:/$testCmd->{'funcjarPath'}/g;
+ $cmd =~ s/:RUNID:/$testCmd->{'UID'}/g;
+ $cmd =~ s/:PIGHARNESS:/$ENV{PIG_HARNESS_ROOT}/g;
+ $cmd =~ s/:USRHOMEPATH:/$testCmd->{'userhomePath'}/g;
+ $cmd =~ s/:SCRIPTHOMEPATH:/$testCmd->{'scriptPath'}/g;
+ $cmd =~ s/:DBUSER:/$testCmd->{'dbuser'}/g;
+ $cmd =~ s/:DBNAME:/$testCmd->{'dbdb'}/g;
+ $cmd =~ s/:LOCALINPATH:/$testCmd->{'localinpathbase'}/g;
+ $cmd =~ s/:LOCALOUTPATH:/$testCmd->{'localoutpathbase'}/g;
+ $cmd =~ s/:BMPATH:/$testCmd->{'benchmarkPath'}/g;
+ $cmd =~ s/:TMP:/$testCmd->{'tmpPath'}/g;
+
+ $cmd =~ s/:LATESTOUTPUTPATH:/$self->{'latestoutputpath'}/g;
+
+ return $cmd;
+}
+########################################################################
+# Sub: generateBenchmark
+# Generate databse benchmark.
+#
+# Parameters:
+# $testCmd -
+# $log -
+#
+# Returns:
+# hash reference containg the test result
+#
+
+sub generateBenchmark
+{
+ my ($self, $testCmd, $log) = @_;
+
+ my %result;
+
+ foreach my $key ('expected_out', 'expected_out_regex', 'expected_err', 'expected_err_regex', 'rc') {
+ if (defined $testCmd->{$key}) {
+ $result{$key} = $testCmd->{$key};
+ }
+
+ }
+ return \%result;
+}
+
+########################################################################
+# Sub: compare
+# Compare the test reuslts to the benchmark results
+#
+# Parameters:
+# None.
+#
+# Returns:
+# the result of the test run. 1 if the test passes.
+
+sub compare
+{
+ my ($self, $testResult, $benchmarkResult, $log) = @_;
+ my $subName = (caller(0))[3];
+ #return 1 if ( $testCmd->{'ignore'}=~ "true" );
+ return 1;
+}
+
+sub compareSAV
+{
+ my ($self, $testResult, $benchmarkResult, $log) = @_;
+ my $subName = (caller(0))[3];
+
+ # IMPORTANT NOTE:
+ # If you are using a regex to compare stdout or stderr
+ # and if the pattern that you are trying to match spans two line
+ # explicitly use '\n' (without the single quotes) in the regex
+
+ if (defined $benchmarkResult->{'rc'} &&
+ ($testResult->{'rc'} != $benchmarkResult->{'rc'})) {
+ print $log "Test and benchmark return code differ:\n";
+ print $log "Test rc = " . $testResult->{'rc'} . "\n";
+ print $log "Expected rc = " . $benchmarkResult->{'rc'} . "\n";
+ return 0;
+ }
+
+ # Check if we are looking for an exact match
+ if (defined $benchmarkResult->{'expected_out'}) {
+ print $log "$0::$subName INFO Checking test result <$testResult->{'stdout'}> " .
+ "as exact match against expected <$benchmarkResult->{'expected_out'}>\n";
+ return $testResult->{'stdout'} eq
+ $benchmarkResult->{'expected_out'};
+
+ } elsif (defined $benchmarkResult->{'expected_out_regex'}) {
+ print $log "$0::$subName INFO Checking test result for regular expression " .
+ "<$benchmarkResult->{'expected_out_regex'}> in " .
+ "<$testResult->{'stdout'}>\n";
+ return $testResult->{'stdout'} =~
+ $benchmarkResult->{'expected_out_regex'};
+
+ } elsif (defined $benchmarkResult->{'expected_err'}) {
+
+ print $log "$0::$subName INFO Checking test result <$testResult->{'stderr'}> " .
+ "as exact match against expected <$benchmarkResult->{'expected_err'}>\n";
+ return $testResult->{'stderr'} eq
+ $benchmarkResult->{'expected_err'};
+
+ } elsif (defined $benchmarkResult->{'expected_err_regex'}) {
+ print $log "$0::$subName INFO Checking test result for regular expression " .
+ "<$benchmarkResult->{'expected_err_regex'}> in " .
+ "<$testResult->{'stderr'}>\n";
+ return $testResult->{'stderr'} =~
+ $benchmarkResult->{'expected_err_regex'};
+
+ } else {
+ return 1;
+ }
+}
+1;
Added: pig/trunk/test/e2e/pig/drivers/Util.pm
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/drivers/Util.pm?rev=1087340&view=auto
==============================================================================
--- pig/trunk/test/e2e/pig/drivers/Util.pm (added)
+++ pig/trunk/test/e2e/pig/drivers/Util.pm Thu Mar 31 15:31:33 2011
@@ -0,0 +1,450 @@
+#!/usr/bin/env perl
+
+############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+###########################################################################
+# Class: Util
+#
+# A collection of helper subroutines.
+#
+
+
+package Util;
+
+##############################################################################
+# Sub: localTime
+#
+# Returns:
+# A string with the local time
+
+sub localTime() {
+
+ my $retval = time();
+
+ my $local_time = gmtime( $retval);
+
+ return $local_time;
+
+}
+
+##############################################################################
+# Sub: formatedTime
+# Returns the time with following format "$mday/$mon/$year $hour:$min:$sec $weekday[$wday]"
+#
+# Returns:
+# formated time
+
+sub formatedTime() {
+
+ my @weekday = ("Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat");
+
+ my $retval = time();
+
+ my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = gmtime(time);
+ $year = $year + 1900;
+ $mon += 1;
+ return "$mday/$mon/$year $hour:$min:$sec $weekday[$wday]\n";
+
+
+}
+
+##############################################################################
+# Sub: execCmd
+# Records the command in the log and prints it to stdout before executing.
+#
+# Paramaters:
+# $log - The log object
+# $subName - The name of the subroutine the message originated at
+# $lineNo - The line number of the subroutine the message originated at
+# $cmd - The command string to execute
+# $msg - A string, the message to print
+# $level - (optional)The logging level for the message: DEBUG, INFO, WARN, ERROR, FATAL
+# defaults to DEBUG.
+#
+# Returns:
+# An array containing the output from the executed command.
+#
+
+sub execCmd() {
+
+ my ( $log, $subName, $lineNo, $cmd, $level ) = @_ ;
+
+ my $count = @_;
+ my $thisSubName = (caller(0))[3];
+
+ #Check for errors in arguments
+ if ( $count < 4 ){
+
+ if ( $log ) {
+ $log->msg( $level, $thisSubName, __LINE__ , "Invalid number of arguments, got $count=( @_ )" );
+
+ } else {
+ print "ERROR: $0 $thisSubName at ".__LINE__."Invalid number of arguments\n";
+
+ }
+ return 1;
+ }
+
+ #Log command, execute commdand, return results
+ $level = "DEBUG" if ( !$level );
+ $log->msg( $level, $subName, $lineNo , "$cmd");
+
+ my @result = `$cmd`;
+
+ $log->msg( $level, $subName, $lineNo , "@cmd") if ( @cmd );
+
+ return @result;
+}
+
+sub getHadoopCmd
+{
+ my ( $properties ) = @_;
+
+ my $subName = (caller(0))[3];
+ my @baseCmd;
+
+ die "$0.$subName: null properties" if (! $properties );
+
+ my $cmd;
+
+ $cmd = $properties->{'gridstack.root'} . "/hadoop/current/bin/hadoop";
+ if ( ! -x "$cmd" ) {
+ print STDERR "\n$0::$subName WARNING: Can't find hadoop command: $cmd\n";
+ $cmd = `which hadoop`;
+ chomp $cmd;
+ print STDERR "$0::$subName WARNING: Instead using command: $cmd\n";
+ }
+ if ( ! -x "$cmd" ) {
+ die "\n$0::$subName FATAL: Hadoop command does not exist: $cmd\n";
+ }
+ push (@baseCmd, $cmd);
+
+ push (@baseCmd, '--config', $properties->{'testconfigpath'}) if defined($properties->{'testconfigpath'});
+
+ return @baseCmd;
+}
+
+
+sub getHiveCmd
+{
+ my ( $properties ) = @_;
+
+ my $subName = (caller(0))[3];
+ my @baseCmd;
+
+ die "$0.$subName: null properties" if (! $properties );
+
+ my $cmd;
+
+ $cmd = $properties->{'hive_bin_location'} . "/hive";
+ if ( ! -x "$cmd" ) {
+ die "\n$0::$subName FATAL: Hive command does not exist: $cmd\n";
+ }
+ push (@baseCmd, $cmd);
+
+# push (@baseCmd, '--config', $properties->{'testconfigpath'}) if defined($properties->{'testconfigpath'});
+
+ return @baseCmd;
+}
+
+sub getHowlCmd
+{
+ my ( $properties ) = @_;
+
+ my $subName = (caller(0))[3];
+ my @baseCmd;
+
+ die "$0.$subName: null properties" if (! $properties );
+
+ my $cmd;
+
+ $cmd = $properties->{'howl_bin_location'} . "/howl";
+ if ( ! -x "$cmd" ) {
+ die "\n$0::$subName FATAL: Howl command does not exist: $cmd\n";
+ }
+ push (@baseCmd, $cmd);
+
+ return @baseCmd;
+}
+
+
+
+sub getPigCmd
+{
+ my $subName = (caller(0))[3];
+ my $jarkey = shift;
+ my ( $properties ) = @_;
+ my $isPigSqlEnabled= 0;
+ my @baseCmd;
+ die "$0.$subName: null properties" if (! $properties );
+
+ #UGLY HACK for pig sql support
+ if ( $jarkey =~ /testsql/ ) {
+
+ $isPigSqlEnabled= 1;
+ $jarkey = "testjar";
+
+ }
+
+ my $cmd;
+ if ( $properties->{'use-pig.pl'} ) {
+ # The directive gives that
+ # 1) the 'pig' command will be called, as opposed to java
+ # 2) the conf file has full control over what options and parameters are
+ # passed to pig.
+ # I.e. no parameters should be passed automatically by the script here.
+ #
+ # This allows for testing of the pig script as installed, and for testin of
+ # the pig script's options, including error testing.
+
+
+ $cmd = $properties->{'gridstack.root'} . "/pig/" . $properties->{'pigTestBuildName'} . "/bin/pig";
+ if ( ! -x "$cmd" ) {
+ print STDERR "\n$0::$subName WARNING: Can't find pig command: $cmd\n";
+ $cmd = `which pig`;
+ chomp $cmd;
+ print STDERR "$0::$subName WARNING: Instead using command: $cmd\n";
+ }
+ die "\n$0::$subName FATAL: Pig command does not exist: $cmd\n" if ( ! -x $cmd );
+ push (@baseCmd, $cmd );
+
+ if ( $properties->{'use-pig.pl'} eq 'raw' ) { # add _no_ arguments automatically
+ # !!!
+ return @baseCmd;
+ }
+
+ } else {
+ $cmd="java";
+
+ # Set JAVA options
+
+ # User can provide only one of
+ # (-c <cluster>) OR (-testjar <jar> -testconfigpath <path>)
+ # "-c <cluster>" is allowed only in non local mode
+ if(defined($properties->{'cluster.name'})) {
+ # use provided cluster
+ @baseCmd = ($cmd, '-c', $properties->{'cluster.name'});
+ } else {
+
+ die "\n$0::$subName FATAL: The jar file name must be passed in at the command line or defined in the configuration file\n" if ( !defined( $properties->{$jarkey} ) );
+ die "\n$0::$subName FATAL: The jar file does not exist.\n" . $properties->{$jarkey}."\n" if ( ! -e $properties->{$jarkey} );
+
+ # use user provided jar
+ my $classpath;
+
+ if (defined $properties->{'jythonjar'}) {
+ $classpath = "$classpath:" . $properties->{'jythonjar'};
+ }
+ if( $properties->{'exectype'} eq "local") {
+ # in local mode, we should not use
+ # any hadoop-site.xml
+ $classpath= "$classpath:" . $properties->{$jarkey};
+ $classpath= "$classpath:$properties->{'classpath'}" if ( defined( $properties->{'classpath'} ) );
+ @baseCmd = ($cmd, '-cp', $classpath, '-Xmx1024m');
+
+ } else {
+
+ # non local mode, we also need to specify
+ # location of hadoop-site.xml
+ die "\n$0::$subName FATAL: The hadoop configuration file name must be passed in at the command line or defined in the configuration file\n"
+ if ( !defined( $properties->{'testconfigpath'} ) );
+ die "\n$0::$subName FATAL $! " . $properties->{'testconfigpath'}."\n\n"
+ if (! -e $properties->{'testconfigpath'} );
+
+ $classpath= "$classpath:" . $properties->{$jarkey}.":".$properties->{'testconfigpath'};
+ $classpath= "$classpath:$properties->{'classpath'}" if ( defined( $properties->{'classpath'} ) );
+ $classpath= "$classpath:$properties->{'howl.jar'}" if ( defined( $properties->{'howl.jar'} ) );
+ @baseCmd = ($cmd, '-cp', $classpath );
+ }
+ }
+
+ # sets the queue, for exampel "grideng"
+ if(defined($properties->{'queue'})) {
+ push( @baseCmd,'-Dmapred.job.queue.name='.$properties->{'queue'});
+ }
+
+ if(defined($properties->{'additionaljars'})) {
+ push( @baseCmd,'-Dpig.additional.jars='.$properties->{'additionaljars'});
+ }
+
+ if( ( $isPigSqlEnabled == 1 ) ){
+
+ if(defined($properties->{'metadata.uri'})) {
+ push( @baseCmd, '-Dmetadata.uri='.$properties->{'metadata.uri'});
+ }
+
+ if(defined($properties->{'metadata.impl'})) {
+ push( @baseCmd, '-Dmetadata.impl='.$properties->{'metadata.impl'});
+ }else{
+ push( @baseCmd, '-Dmetadata.impl=org.apache.hadoop.owl.pig.metainterface.OwlPigMetaTables');
+ }
+ }
+
+ # Add howl support
+ if(defined($properties->{'howl.metastore.uri'})) {
+ push( @baseCmd, '-Dhowl.metastore.uri='.$properties->{'howl.metastore.uri'});
+ }
+
+ # Set local mode property
+ # if ( defined($properties->{'exectype'}) && $properties->{'exectype'}=~ "local" ) {
+ # Removed above 'if...' for Pig 8.
+ my $java=`which java`;
+ my $version=`file $java`;
+ if ( $version =~ '32-bit' ){
+ push(@baseCmd,'-Djava.library.path='.$ENV{HADOOP_HOME}.'/lib/native/Linux-i386-32');
+ } else {
+ push(@baseCmd,'-Djava.library.path='.$ENV{HADOOP_HOME}.'/lib/native/Linux-amd64-64');
+ }
+ # }
+
+
+ # Add user provided java options if they exist
+ if (defined($properties->{'java_params'})) {
+ push(@baseCmd, @{$properties->{'java_params'}});
+ }
+
+ if(defined($properties->{'hod'})) {
+ push( @baseCmd, '-Dhod.server=');
+ }
+
+ # sets the permissions on the jobtracker for the logs
+ push( @baseCmd,'-Dmapreduce.job.acl-view-job=*');
+
+
+ # Add Main
+ push(@baseCmd, 'org.apache.pig.Main');
+
+ # Set local mode PIG option
+ if ( defined($properties->{'exectype'}) && $properties->{'exectype'}=~ "local" ) {
+ push(@baseCmd, '-x');
+ push(@baseCmd, 'local');
+ }
+
+ # Set Pig SQL options
+ if( ( $isPigSqlEnabled == 1 ) && defined($properties->{'metadata.uri'})) {
+
+ if ( defined($properties->{'testoutpath'}) ) {
+ push( @baseCmd, '-u' );
+ push( @baseCmd, $properties->{'testoutpath'} );
+ }
+
+ push( @baseCmd, '-s' );
+ push( @baseCmd, '-f' );
+ }
+
+ } # end else of if use-pig.pl
+
+
+ # Add -latest or -useversion
+ if ( $cmd =~ 'pig$' ) {
+ # Add -latest, or -useversion if 'current' is not target build
+ if ( defined($properties->{'pigTestBuildName'})) {
+ if ($properties->{'pigTestBuildName'} eq 'latest') {
+ push(@baseCmd, '-latest');
+ } elsif ($properties->{'pigTestBuildName'} ne 'current') {
+ push(@baseCmd, '-useversion', "$properties->{'pigTestBuildName'}");
+ }
+ }
+ } elsif ( $cmd =~ 'java' ) {
+
+ # is this ever used: ???
+ # Add latest if it's there
+ if (defined($properties->{'latest'})) {
+ push(@baseCmd, '-latest');
+ }
+ }
+
+ return @baseCmd;
+}
+
+
+sub getBasePigSqlCmd
+{
+
+ my $subName = (caller(0))[3];
+
+ Util::getPigCmd( 'testsql', @_ );
+
+}
+
+sub getBasePigCmd
+{
+
+ my $subName = (caller(0))[3];
+
+ Util::getPigCmd( 'testjar', @_ );
+
+}
+
+sub getLatestBasePigCmd
+{
+
+ my $subName = (caller(0))[3];
+
+ Util::getPigCmd( 'latesttestjar', @_ );
+
+}
+
+
+sub getBenchmarkBasePigCmd
+{
+ my $subName = (caller(0))[3];
+ my ( $properties ) = @_;
+
+ Util::getPigCmd( 'benchmarkjar', @_ );
+
+}
+
+sub setLocale
+{
+ my $locale= shift;
+# $locale = "en_US.UTF-8" if ( !$locale );
+$locale = "ja_JP.utf8" if ( !$locale );
+ $ENV[LC_CTYPE]="$locale";
+ $ENV[LC_NUMERIC]="$locale";
+ $ENV[LC_TIME]="$locale";
+ $ENV[LC_COLLATE]="$locale";
+ $ENV[LC_MONETARY]="$locale";
+ $ENV[LC_MESSAGES]="$locale";
+ $ENV[LC_PAPER]="$locale";
+ $ENV[LC_NAME]="$locale";
+ $ENV[LC_ADDRESS]="$locale";
+ $ENV[LC_TELEPHONE]="$locale";
+ $ENV[LC_MEASUREMENT]="$locale";
+ $ENV[LC_IDENTIFICATION]="$locale";
+}
+
+sub getLocaleCmd
+{
+ my $locale= shift;
+ $locale = "en_US.UTF-8" if ( !$locale );
+
+ return "export LC_CTYPE=\"$locale\";"
+ ."export LC_NUMERIC=\"$locale\";"
+ ."export LC_TIME=\"$locale\";"
+ ."export LC_COLLATE=\"$locale\";"
+ ."export LC_MONETARY=\"$locale\";"
+ ."export LC_MESSAGES=\"$locale\";"
+ ."export LC_PAPER=\"$locale\";"
+ ."export LC_NAME=\"$locale\";"
+ ."export LC_ADDRESS=\"$locale\";"
+ ."export LC_TELEPHONE=\"$locale\";"
+ ."export LC_MEASUREMENT=\"$locale\";"
+ ."export LC_IDENTIFICATION=\"$locale\"";
+}
+1;
Added: pig/trunk/test/e2e/pig/lib/hadoop-examples.jar
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/lib/hadoop-examples.jar?rev=1087340&view=auto
==============================================================================
Binary file - no diff available.
Propchange: pig/trunk/test/e2e/pig/lib/hadoop-examples.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: pig/trunk/test/e2e/pig/lib/hadoop-streaming.jar
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/lib/hadoop-streaming.jar?rev=1087340&view=auto
==============================================================================
Binary file - no diff available.
Propchange: pig/trunk/test/e2e/pig/lib/hadoop-streaming.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: pig/trunk/test/e2e/pig/paramfiles/params_3
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/paramfiles/params_3?rev=1087340&view=auto
==============================================================================
--- pig/trunk/test/e2e/pig/paramfiles/params_3 (added)
+++ pig/trunk/test/e2e/pig/paramfiles/params_3 Thu Mar 31 15:31:33 2011
@@ -0,0 +1,3 @@
+# this is a comment
+fname = studenttab10k
+unusedkey = unusedvalueihope