You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2014/09/20 00:23:54 UTC
svn commit: r1626356 - in /pig/trunk: ./ test/e2e/harness/ test/e2e/pig/
test/e2e/pig/conf/ test/e2e/pig/drivers/ test/e2e/pig/tests/
test/e2e/pig/udfs/java/
Author: rohini
Date: Fri Sep 19 22:23:54 2014
New Revision: 1626356
URL: http://svn.apache.org/r1626356
Log:
PIG-4186: Fix e2e run against new build of pig and some enhancements (rohini)
Added:
pig/trunk/test/e2e/harness/xmlReport.pl (with props)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/test/e2e/harness/TestDriver.pm
pig/trunk/test/e2e/pig/build.xml
pig/trunk/test/e2e/pig/conf/default.conf
pig/trunk/test/e2e/pig/conf/local.conf
pig/trunk/test/e2e/pig/conf/tez.conf
pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm
pig/trunk/test/e2e/pig/tests/nightly.conf
pig/trunk/test/e2e/pig/udfs/java/build.xml
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1626356&r1=1626355&r2=1626356&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Sep 19 22:23:54 2014
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-4186: Fix e2e run against new build of pig and some enhancements (rohini)
+
PIG-3838: Organize tez code into subpackages (rohini)
PIG-4069: Limit reduce task should start as soon as one map task finishes (rohini)
Modified: pig/trunk/test/e2e/harness/TestDriver.pm
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/harness/TestDriver.pm?rev=1626356&r1=1626355&r2=1626356&view=diff
==============================================================================
--- pig/trunk/test/e2e/harness/TestDriver.pm (original)
+++ pig/trunk/test/e2e/harness/TestDriver.pm Fri Sep 19 22:23:54 2014
@@ -717,7 +717,7 @@ sub runTestGroup() {
$testStatuses->{$testName} = $failedStr;
}
- $msg .= "\nEnding test $testName at " . time ."\n";
+ $msg .= " at " . time . "\nEnding test $testName at " . time ."\n";
print $subLog $msg;
$duration = $endTime - $beginTime;
$dbinfo{'duration'} = $duration;
Added: pig/trunk/test/e2e/harness/xmlReport.pl
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/harness/xmlReport.pl?rev=1626356&view=auto
==============================================================================
--- pig/trunk/test/e2e/harness/xmlReport.pl (added)
+++ pig/trunk/test/e2e/harness/xmlReport.pl Fri Sep 19 22:23:54 2014
@@ -0,0 +1,138 @@
+#!/usr/bin/env perl
+use Sys::Hostname;
+
+use strict;
+
+use File::Path;
+use IO::Handle;
+
+#
+# Set Path for perl Libraries
+#
+unshift( @INC, ".");
+
+
+########################################################################
+# Sub: testcase
+# Prints the result for a test case.
+#
+# Parameters:
+# $class - the test file
+# $test - the test name
+# $time - the the total amount of time it took to run a test
+#
+# Returns:
+# None
+#
+sub getTestcase($$$$$$){
+
+ my ( $class_name, $group_name, $test_name, $time, $status, $msg )=@_;
+ my $result = "";
+
+ if ( $status =~ "passed" ){
+ $result = "\n<testcase classname=\"$class_name\" name=\"$test_name\" time=\"$time\"/>";
+
+ } elsif ( $status =~ "skipped" ){
+ $result = "\n<testcase classname=\"$class_name\" name=\"$test_name\" time=\"0\">"
+ . "\n<skipped type=\"$status\">$msg</skipped>"
+ . "\n</testcase>"
+ ;
+ } else {
+ $result = "\n<testcase classname=\"$class_name\" name=\"$test_name\" time=\"$time\">"
+ . "\n<error type=\"$status\">$msg</error>"
+ . "\n</testcase>"
+ ;
+ } return $result;
+}
+
+
+sub printXmlReport($) {
+
+ my $host = Sys::Hostname::hostname();
+ my $tmpFileName = shift;
+ my $reportFileName = shift;
+
+ my $testcases = "";
+ my $total_time= 0;
+ my $testnameSuffix = "";
+
+ if ($tmpFileName =~ m/-local/) {
+ $testnameSuffix = "_local";
+ }
+
+ my $passedCount = 0;
+ my $failureCount = 0;
+ my $errorCount = 0;
+ my $skippedCount = 0;
+ my $totalCount = 0;
+
+ my %test2starttime = { '', 0 };
+
+ open( IN, "$tmpFileName")
+ || die "Could not open $tmpFileName\n";
+
+ while(<IN>){
+
+ my $line = $_;
+ # e.g.: "Beginning test Checkin_1 at 1346855793"
+ if ( $line =~ m/Beginning test (.*) at (.*)/ ) {
+ # put "test - start time" pair to the hash:
+ $test2starttime{ $1 } = $2;
+ } else {
+ if ( $line =~ "TestDriver::run" ) {
+ my $duration = 0;
+ if ( $line =~ m/TestDriver::run.*Failed to run test (.*) <(.*)/ ) {
+ # ERROR TestDriver::run at : 470 Failed to run test ClassResolution_1 <Failed running ./out/pigtest/hadoopqa/hadoopqa.1327755958/ClassResolution_1_benchmark.pig
+ #print "test aborted: $line";
+ $testcases .= getTestcase ( $1, "group", $1 . $testnameSuffix, 0, "aborted", $2 );
+ $errorCount++;
+ } elsif ( $line =~ m/TestDriver::run.*Test (.*) SUCCEEDED at (.*)/ ) {
+ # INFO: TestDriver::run() at 444:Test Unicode_cmdline_1 SUCCEEDED at 1327751873
+ #print "test passed: $1 $2 line: $line";
+ $passedCount++;
+ $duration = $2 - $test2starttime{ $1 };
+ $testcases .= getTestcase ( $1, "group", $1 . $testnameSuffix, $duration, "passed", "" );
+ } elsif ( $line =~ m/TestDriver::run.*Test (.*) FAILED at (.*)/ ) {
+ $failureCount++;
+ $duration = $2 - $test2starttime{ $1 };
+ $testcases .= getTestcase ( $1, "group", $1 . $testnameSuffix, $duration, "failed", "" );
+ #print "test failed: $1 $2 line: $line";
+ } elsif ($line =~ "Running TEST GROUP") {
+ next;
+ } elsif ($line =~ m/TestDriver::run.*Test (.*) SKIPPED at (.*)/) {
+ # INFO: TestDriver::run() at 444:Test StreamingLocalErrors_1 SKIPPED at 1327923441
+ $skippedCount++;
+ $duration = $2 - $test2starttime{ $1 };
+ $testcases .= getTestcase ( $1, "group", $1 . $testnameSuffix, $duration, "skipped", "" );
+ } else {
+ print STDERR "Ignored line: $line";
+ next;
+ }
+ #$total_time= $total_time + $time;
+ $totalCount++;
+ }
+ }
+
+ }
+ close(IN);
+ #Report
+ my $host = Sys::Hostname::hostname();
+ my $run_name = "e2e tests";
+ my $report=
+ '<?xml version="1.0" encoding="UTF-8" ?>'
+ . "\n<testsuite errors=\"$errorCount\" failures=\"$failureCount\" skips=\"$skippedCount\" hostname=\"$host\" name=\"$run_name\" tests=\"$totalCount\" time=\"$total_time\">"
+ . "\n$testcases"
+ . "\n</testsuite>"
+ . "\n";
+ print $report;
+}
+
+
+if (!defined( $ARGV[0] )) {
+ die "No input log file specified\n";
+}
+
+printXmlReport($ARGV[0]);
+
+
+
Propchange: pig/trunk/test/e2e/harness/xmlReport.pl
------------------------------------------------------------------------------
svn:executable = *
Modified: pig/trunk/test/e2e/pig/build.xml
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/build.xml?rev=1626356&r1=1626355&r2=1626356&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/build.xml (original)
+++ pig/trunk/test/e2e/pig/build.xml Fri Sep 19 22:23:54 2014
@@ -18,15 +18,14 @@
<project name="TestHarnessPigTests" default="test">
<property name="pig.dir" value="${basedir}/../../.."/>
- <property name="ivy.dir" location="${pig.dir}/ivy" />
+ <property name="pig.base.dir" value="${basedir}/../../.."/>
+ <property name="ivy.dir" location="${pig.base.dir}/ivy" />
<loadproperties srcfile="${ivy.dir}/libraries.properties"/>
- <property name="jython.jar"
- value="${pig.dir}/build/ivy/lib/Pig/jython-standalone-${jython.version}.jar"/>
- <property name="jruby.jar"
- value="${pig.dir}/build/ivy/lib/Pig/jruby-complete-${jruby.version}.jar"/>
+ <property name="piggybank.jar"
+ value="${pig.base.dir}/contrib/piggybank/java/piggybank.jar"/>
<property name="hive.lib.dir"
- value="${pig.dir}/build/ivy/lib/Pig"/>
+ value="${pig.base.dir}/build/ivy/lib/Pig"/>
<condition property="hive.hadoop.shims.version" value="0.23" else="0.20S">
<equals arg1="${hadoopversion}" arg2="23" />
@@ -279,6 +278,7 @@
<env key="HADOOP_CONF_DIR" value="${harness.cluster.conf}"/>
<env key="PIG_USE_PYTHON" value="${harness.use.python}"/>
<env key="PH_CLUSTER_BIN" value="${harness.cluster.bin}"/>
+ <env key="PH_PIGGYBANK_JAR" value="${piggybank.jar}"/>
<env key="PH_HIVE_LIB_DIR" value="${hive.lib.dir}"/>
<env key="PH_HIVE_VERSION" value="${hive.version}"/>
<env key="PH_HIVE_SHIMS_VERSION" value="${hive.hadoop.shims.version}"/>
Modified: pig/trunk/test/e2e/pig/conf/default.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/conf/default.conf?rev=1626356&r1=1626355&r2=1626356&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/conf/default.conf (original)
+++ pig/trunk/test/e2e/pig/conf/default.conf Fri Sep 19 22:23:54 2014
@@ -49,7 +49,7 @@ $cfg = {
, 'testconfigpath' => "$ENV{HADOOP_CONF_DIR}"
, 'funcjarPath' => "$ENV{PH_ROOT}/lib/java"
, 'paramPath' => "$ENV{PH_ROOT}/paramfiles"
- , 'piggybankjarPath' => "$ENV{PH_PIG}/contrib/piggybank/java"
+ , 'piggybankjarPath' => "$ENV{PH_PIGGYBANK_JAR}"
, 'pigpath' => "$ENV{PH_PIG}"
, 'oldpigpath' => "$ENV{PH_OLDPIG}"
, 'hcatbin' => "$ENV{HCAT_BIN}"
Modified: pig/trunk/test/e2e/pig/conf/local.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/conf/local.conf?rev=1626356&r1=1626355&r2=1626356&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/conf/local.conf (original)
+++ pig/trunk/test/e2e/pig/conf/local.conf Fri Sep 19 22:23:54 2014
@@ -46,7 +46,7 @@ $cfg = {
, 'testconfigpath' => "$ENV{PH_CLUSTER}/conf/"
, 'funcjarPath' => "$ENV{PH_ROOT}/lib/java"
, 'paramPath' => "$ENV{PH_ROOT}/paramfiles"
- , 'piggybankjarPath' => "$ENV{PH_PIG}/contrib/piggybank/java"
+ , 'piggybankjarPath' => "$ENV{PH_PIGGYBANK_JAR}"
, 'pigpath' => "$ENV{PH_PIG}"
, 'oldpigpath' => "$ENV{PH_OLDPIG}"
, 'exectype' => 'local'
Modified: pig/trunk/test/e2e/pig/conf/tez.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/conf/tez.conf?rev=1626356&r1=1626355&r2=1626356&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/conf/tez.conf (original)
+++ pig/trunk/test/e2e/pig/conf/tez.conf Fri Sep 19 22:23:54 2014
@@ -49,7 +49,7 @@ $cfg = {
, 'testconfigpath' => "$ENV{HADOOP_CONF_DIR}"
, 'funcjarPath' => "$ENV{PH_ROOT}/lib/java"
, 'paramPath' => "$ENV{PH_ROOT}/paramfiles"
- , 'piggybankjarPath' => "$ENV{PH_PIG}/contrib/piggybank/java"
+ , 'piggybankjarPath' => "$ENV{PH_PIGGYBANK_JAR}"
, 'pigpath' => "$ENV{PH_PIG}"
, 'oldpigpath' => "$ENV{PH_OLDPIG}"
, 'hcatbin' => "$ENV{HCAT_BIN}"
Modified: pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm?rev=1626356&r1=1626355&r2=1626356&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm (original)
+++ pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm Fri Sep 19 22:23:54 2014
@@ -73,7 +73,7 @@ sub replaceParameters
$cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g;
$cmd =~ s/:OUTPATH:/$outfile/g;
$cmd =~ s/:FUNCPATH:/$testCmd->{'funcjarPath'}/g;
- $cmd =~ s/:PIGGYBANKPATH:/$testCmd->{'piggybankjarPath'}/g;
+ $cmd =~ s/:PIGGYBANKJAR:/$testCmd->{'piggybankjarPath'}/g;
$cmd =~ s/:PIGPATH:/$testCmd->{'pigpath'}/g;
$cmd =~ s/:RUNID:/$testCmd->{'UID'}/g;
$cmd =~ s/:USRHOMEPATH:/$testCmd->{'userhomePath'}/g;
Modified: pig/trunk/test/e2e/pig/tests/nightly.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/nightly.conf?rev=1626356&r1=1626355&r2=1626356&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/nightly.conf (original)
+++ pig/trunk/test/e2e/pig/tests/nightly.conf Fri Sep 19 22:23:54 2014
@@ -4719,7 +4719,7 @@ store C into ':OUTPATH:';\,
# Simplest merge-sparse-join.
{
'num' => 1,
- 'pig' => q\register :PIGGYBANKPATH:/piggybank.jar
+ 'pig' => q\register :PIGGYBANKJAR:
a = load ':INPATH:/singlefile/studenttab10k';
b = load ':INPATH:/singlefile/votertab10k';
c = order a by $0;
@@ -4740,7 +4740,7 @@ store C into ':OUTPATH:';\,
# Merge-sparse-join with left-side filter
{
'num' => 2,
- 'pig' => q\register :PIGGYBANKPATH:/piggybank.jar
+ 'pig' => q\register :PIGGYBANKJAR:
a = load ':INPATH:/singlefile/studenttab10k';
b = load ':INPATH:/singlefile/votertab10k';
c = order a by $0;
@@ -4763,7 +4763,7 @@ store C into ':OUTPATH:';\,
# Merge-sparse-join with right-side filter
{
'num' => 3,
- 'pig' => q\register :PIGGYBANKPATH:/piggybank.jar
+ 'pig' => q\register :PIGGYBANKJAR:
a = load ':INPATH:/singlefile/studenttab10k';
b = load ':INPATH:/singlefile/votertab10k';
c = order a by $0;
@@ -4787,7 +4787,7 @@ store C into ':OUTPATH:';\,
# Merge-sparse-join with key as expression
{
'num' => 4,
- 'pig' => q\register :PIGGYBANKPATH:/piggybank.jar
+ 'pig' => q\register :PIGGYBANKJAR:
a = load ':INPATH:/singlefile/studenttab10k';
b = load ':INPATH:/singlefile/votertab10k';
c = order a by $0,$1;
@@ -4808,7 +4808,7 @@ store C into ':OUTPATH:';\,
# Merge-sparse-join with nulls in keys and data.
{
'num' => 5,
- 'pig' => q\register :PIGGYBANKPATH:/piggybank.jar
+ 'pig' => q\register :PIGGYBANKJAR:
a = load ':INPATH:/singlefile/studentnulltab10k';
b = load ':INPATH:/singlefile/voternulltab10k';
c = order a by $0;
@@ -4829,7 +4829,7 @@ store C into ':OUTPATH:';\,
# Merge-sparse-join with join on numeric key
{
'num' => 6,
- 'pig' => q\register :PIGGYBANKPATH:/piggybank.jar
+ 'pig' => q\register :PIGGYBANKJAR:
a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:int, gpa:float);
b = load ':INPATH:/singlefile/votertab10k'as (name:chararray, age:int, reg:chararray, contrib:float);
c = order a by age;
Modified: pig/trunk/test/e2e/pig/udfs/java/build.xml
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/udfs/java/build.xml?rev=1626356&r1=1626355&r2=1626356&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/udfs/java/build.xml (original)
+++ pig/trunk/test/e2e/pig/udfs/java/build.xml Fri Sep 19 22:23:54 2014
@@ -26,10 +26,10 @@
<!-- <fileset dir="lib">
<include name="**/*.jar"/>
</fileset>-->
- <fileset dir="${pig.dir}/build/ivy/lib/Pig">
+ <fileset dir="${pig.base.dir}/build/ivy/lib/Pig">
<include name="*.jar"/>
</fileset>
- <fileset dir="${pig.dir}">
+ <fileset dir="${pig.base.dir}">
<include name="pig*-core-*.jar"/>
</fileset>
</path>