svn commit: r1667642 - in /pig/trunk: CHANGES.txt test/e2e/pig/build.xml test/e2e/pig/drivers/TestDriverPig.pm

[da...@apache.org]

Author: daijy
Date: Thu Mar 19 00:14:11 2015
New Revision: 1667642

URL: http://svn.apache.org/r1667642
Log:
PIG-4461: Use benchmarks for Windows Pig e2e tests

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/test/e2e/pig/build.xml
    pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1667642&r1=1667641&r2=1667642&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Mar 19 00:14:11 2015
@@ -54,6 +54,8 @@ PIG-4333: Split BigData tests into multi
  
 BUG FIXES
 
+PIG-4461: Use benchmarks for Windows Pig e2e tests (nmaheshwari via daijy)
+
 PIG-4463: AvroMapWrapper still leaks Avro data types and AvroStorageDataConversionUtilities do not handle
  Pig maps (rdsr via daijy)
 

Modified: pig/trunk/test/e2e/pig/build.xml
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/build.xml?rev=1667642&r1=1667641&r2=1667642&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/build.xml (original)
+++ pig/trunk/test/e2e/pig/build.xml Thu Mar 19 00:14:11 2015
@@ -312,6 +312,7 @@
       <env key="FORK_FACTOR_FILE" value="${fork.factor.conf.file}"/>
       <env key="HADOOP_MAPRED_LOCAL_DIR" value="${hadoop.mapred.local.dir}"/>
       <env key="E2E_DEBUG" value="${e2e.debug}"/>
+      <env key="SORT_BENCHMARKS" value="${sort.benchmarks}"/>
 
       <arg value="./test_harness.pl"/>
       <arg line="${tests.to.run} ${tests.suites}"/>

Modified: pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm?rev=1667642&r1=1667641&r2=1667642&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm (original)
+++ pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm Thu Mar 19 00:14:11 2015
@@ -595,8 +595,8 @@ sub postProcessSingleOutputFile
     # Build command to:
     # 1. Combine part files
     my $fppCmd;
-    if(Util::isWindows()) {
-        my $delCmd = "del \"$localdir\\*.crc\"";
+    if(Util::isWindows()||Util::isCygwin()) {
+        my $delCmd = "del \"$localdir\\*.crc\" 2>NUL";
         print $log "$delCmd\n";
         system($delCmd);
         $fppCmd = "cat $localdir\\map* $localdir\\part* 2>NUL";
@@ -614,6 +614,11 @@ sub postProcessSingleOutputFile
     
     $fppCmd .= " > $localdir/out_original";
     
+    #Need slashes to be consistent for windows
+    if (Util::isWindows() || Util::isCygwin()) {
+        $fppCmd =~ s/\\/\//g;
+    }
+    
     # run command
     print $log "$fppCmd\n";
     system($fppCmd);
@@ -623,6 +628,25 @@ sub postProcessSingleOutputFile
     print $log join(" ", @sortCmd) . "\n";
     IPC::Run::run(\@sortCmd, '>', "$localdir/out_sorted") or die "Sort for benchmark comparison failed on $localdir/out_original";
 
+    # Remove extra \r from $localdir/out_sorted for Windows benchmark
+    if(Util::isWindows()||Util::isCygwin()) {
+        my $tmpfile = "$localdir/out_sorted.tmp";
+        link("$localdir/out_sorted", $tmpfile) or
+            die "Unable to create temporary file $tmpfile, $!\n";
+        unlink("$localdir/out_sorted") or
+            die "Unable to unlink file $localdir/out_sorted, $!\n";
+        open(IFH, "< $tmpfile") or
+            die "Unable to open file $tmpfile, $!\n";
+        open(OFH, "> $localdir/out_sorted") or
+            die "Unable to open file $localdir/out_sorted, $!\n";
+        while(<IFH>) {
+            $_ =~ s/\r$//g;
+            print OFH $_;
+        }
+        close(OFH);
+        close(IFH);
+        unlink($tmpfile);
+    }
     return "$localdir/out_sorted";
 }
 
@@ -936,6 +960,17 @@ sub compareSingleOutput
 {
     my ($self, $testResult, $testOutput, $benchmarkOutput, $log) = @_;
 
+    if ($ENV{'SORT_BENCHMARKS'} eq 'true'){
+        # Sort the benchmark Output.
+        my $benchmarkOutput_new = $benchmarkOutput.'_new';
+        my @sortCmd = ('sort', "$benchmarkOutput");
+        print $log join(" ", @sortCmd) . "\n";
+        IPC::Run::run(\@sortCmd, '>', "$benchmarkOutput_new") or die "Sort for benchmark ouput failed on $benchmarkOutput_new";
+        my @renameCmd = ('mv', "$benchmarkOutput_new" , "$benchmarkOutput");
+        print $log join(" ", @renameCmd) . "\n";
+        IPC::Run::run(\@renameCmd, \undef, $log, $log) or die "Rename command failed";
+    }
+
     # cksum the the two files to see if they are the same
     my ($testChksm, $benchmarkChksm);
     IPC::Run::run((['cat', $testOutput], '|', ['cksum']), \$testChksm,