You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/10/31 08:31:17 UTC
svn commit: r1635710 - in /pig/branches/branch-0.14: ./ test/e2e/pig/conf/
test/e2e/pig/tests/ test/e2e/pig/udfs/java/
Author: daijy
Date: Fri Oct 31 07:31:17 2014
New Revision: 1635710
URL: http://svn.apache.org/r1635710
Log:
PIG-4258: Fix several e2e tests on Windows
Modified:
pig/branches/branch-0.14/CHANGES.txt
pig/branches/branch-0.14/test/e2e/pig/conf/local.conf
pig/branches/branch-0.14/test/e2e/pig/conf/rpm.conf
pig/branches/branch-0.14/test/e2e/pig/conf/tez.conf
pig/branches/branch-0.14/test/e2e/pig/tests/grunt.conf
pig/branches/branch-0.14/test/e2e/pig/tests/macro.conf
pig/branches/branch-0.14/test/e2e/pig/tests/multiquery.conf
pig/branches/branch-0.14/test/e2e/pig/tests/streaming.conf
pig/branches/branch-0.14/test/e2e/pig/udfs/java/build.xml
Modified: pig/branches/branch-0.14/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/CHANGES.txt?rev=1635710&r1=1635709&r2=1635710&view=diff
==============================================================================
--- pig/branches/branch-0.14/CHANGES.txt (original)
+++ pig/branches/branch-0.14/CHANGES.txt Fri Oct 31 07:31:17 2014
@@ -97,6 +97,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-4258: Fix several e2e tests on Windows (daijy)
+
PIG-4256: Fix StreamingPythonUDFs e2e test failure on Windows (daijy)
PIG-4166: Collected group drops last record when combined with merge join (bridiver via daijy)
Modified: pig/branches/branch-0.14/test/e2e/pig/conf/local.conf
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/test/e2e/pig/conf/local.conf?rev=1635710&r1=1635709&r2=1635710&view=diff
==============================================================================
--- pig/branches/branch-0.14/test/e2e/pig/conf/local.conf (original)
+++ pig/branches/branch-0.14/test/e2e/pig/conf/local.conf Fri Oct 31 07:31:17 2014
@@ -15,7 +15,7 @@
# limitations under the License.
my $me = `whoami`;
-chomp $me;
+$me =~ s/[^a-zA-Z0-9]*//g;
# The contents of this file can be rewritten to fit your installation.
# Also, you can define the following environment variables and set things up as in the test setup
@@ -40,7 +40,7 @@ $cfg = {
#TEST
, 'benchmarkPath' => "$ENV{PH_OUT}/benchmarks"
, 'scriptPath' => "$ENV{PH_ROOT}/libexec"
- , 'tmpPath' => '/tmp/pigtest'
+ , 'tmpPath' => 'tmp/pigtest'
#PIG
, 'testconfigpath' => "$ENV{PH_CLUSTER}/conf/"
Modified: pig/branches/branch-0.14/test/e2e/pig/conf/rpm.conf
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/test/e2e/pig/conf/rpm.conf?rev=1635710&r1=1635709&r2=1635710&view=diff
==============================================================================
--- pig/branches/branch-0.14/test/e2e/pig/conf/rpm.conf (original)
+++ pig/branches/branch-0.14/test/e2e/pig/conf/rpm.conf Fri Oct 31 07:31:17 2014
@@ -15,7 +15,7 @@
# limitations under the License.
my $me = `whoami`;
-chomp $me;
+$me =~ s/[^a-zA-Z0-9]*//g;
# The contents of this file can be rewritten to fit your installation.
# Also, you can define the following environment variables and set things up as in the test setup
@@ -43,14 +43,13 @@ $cfg = {
#TEST
, 'benchmarkPath' => "$ENV{PH_OUT}/benchmarks"
, 'scriptPath' => "$ENV{PH_ROOT}/libexec"
- , 'tmpPath' => '/tmp/pigtest'
+ , 'tmpPath' => 'tmp/pigtest'
#PIG
, 'testconfigpath' => "$ENV{HADOOP_CONF_DIR}"
, 'funcjarPath' => "$ENV{PH_ROOT}/lib/java"
- , 'piggybankjarPath' => "/usr/lib/pig"
+ , 'piggybankjarPath' => "/usr/lib/pig/lib/piggybank.jar"
, 'paramPath' => "$ENV{PH_ROOT}/paramfiles"
- , 'piggybankjarPath' => "/usr/lib/pig"
, 'pigpath' => "/usr"
, 'oldpigpath' => "$ENV{PH_OLDPIG}"
, 'hcatbin' => "$ENV{HCAT_BIN}"
Modified: pig/branches/branch-0.14/test/e2e/pig/conf/tez.conf
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/test/e2e/pig/conf/tez.conf?rev=1635710&r1=1635709&r2=1635710&view=diff
==============================================================================
--- pig/branches/branch-0.14/test/e2e/pig/conf/tez.conf (original)
+++ pig/branches/branch-0.14/test/e2e/pig/conf/tez.conf Fri Oct 31 07:31:17 2014
@@ -43,7 +43,7 @@ $cfg = {
#TEST
, 'benchmarkPath' => "$ENV{PH_OUT}/benchmarks"
, 'scriptPath' => "$ENV{PH_ROOT}/libexec"
- , 'tmpPath' => '/tmp/pigtest'
+ , 'tmpPath' => 'tmp/pigtest'
#PIG
, 'testconfigpath' => "$ENV{HADOOP_CONF_DIR}"
Modified: pig/branches/branch-0.14/test/e2e/pig/tests/grunt.conf
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/test/e2e/pig/tests/grunt.conf?rev=1635710&r1=1635709&r2=1635710&view=diff
==============================================================================
--- pig/branches/branch-0.14/test/e2e/pig/tests/grunt.conf (original)
+++ pig/branches/branch-0.14/test/e2e/pig/tests/grunt.conf Fri Oct 31 07:31:17 2014
@@ -44,7 +44,7 @@ $cfg = {
'num' => 2,
'pig' => "pwd",
'execonly' => 'mapred,tez', # don't have a clue what their cwd will be for local mode
- 'expected_out_regex' => "hdfs:",
+ 'expected_out_regex' => "/user",
'rc' => 0
},{
@@ -55,10 +55,8 @@ $cfg = {
},{
'num' => 6,
- 'pig' => q\
-sh touch /bin/bad
-\,
- ,'expected_err_regex' => "Permission denied"
+ 'pig' => "cat nonexist"
+ ,'expected_err_regex' => "does not exist"
,'rc' => 5
},{
Modified: pig/branches/branch-0.14/test/e2e/pig/tests/macro.conf
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/test/e2e/pig/tests/macro.conf?rev=1635710&r1=1635709&r2=1635710&view=diff
==============================================================================
--- pig/branches/branch-0.14/test/e2e/pig/tests/macro.conf (original)
+++ pig/branches/branch-0.14/test/e2e/pig/tests/macro.conf Fri Oct 31 07:31:17 2014
@@ -373,10 +373,16 @@ $cfg = {
A = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
x = test(A);
store x into ':OUTPATH:';#,
+ 'pig_win' => q#define CMD `perl -ne "print $_;"`;
+ define test(in) returns B {
+ $B = stream $in through CMD as (name, age, gpa);
+ }
+
+ A = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
+ x = test(A);
+ store x into ':OUTPATH:';#,
'verify_pig_script' => q#A = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
- define CMD `perl -ne 'print $_;'`;
- B = stream A through CMD as (name, age, gpa);
- store B into ':OUTPATH:';#,
+ store A into ':OUTPATH:';#,
'floatpostprocess' => 1,
'delimiter' => ' '
},
Modified: pig/branches/branch-0.14/test/e2e/pig/tests/multiquery.conf
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/test/e2e/pig/tests/multiquery.conf?rev=1635710&r1=1635709&r2=1635710&view=diff
==============================================================================
--- pig/branches/branch-0.14/test/e2e/pig/tests/multiquery.conf (original)
+++ pig/branches/branch-0.14/test/e2e/pig/tests/multiquery.conf Fri Oct 31 07:31:17 2014
@@ -236,6 +236,14 @@ $cfg = {
C = stream B through CMD2 as (name, age, gpa);
D = JOIN B by name, C by name;
store D into ':OUTPATH:.2'; #,
+ 'pig_win' => q# define CMD1 `perl -ne "print $_;"`;
+ define CMD2 `perl -ne "print $_;"`;
+ A = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
+ B = stream A through CMD1 as (name, age, gpa);
+ store B into ':OUTPATH:.1';
+ C = stream B through CMD2 as (name, age, gpa);
+ D = JOIN B by name, C by name;
+ store D into ':OUTPATH:.2'; #,
'sql' => "select name, age, gpa from studenttab10k;
select A.name, A.age, A.gpa, B.name, B.age, B.gpa
from studenttab10k as A join studenttab10k as B using(name);",
Modified: pig/branches/branch-0.14/test/e2e/pig/tests/streaming.conf
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/test/e2e/pig/tests/streaming.conf?rev=1635710&r1=1635709&r2=1635710&view=diff
==============================================================================
--- pig/branches/branch-0.14/test/e2e/pig/tests/streaming.conf (original)
+++ pig/branches/branch-0.14/test/e2e/pig/tests/streaming.conf Fri Oct 31 07:31:17 2014
@@ -407,6 +407,15 @@ store B into ':OUTPATH:.intermediate';
C = stream B through CMD2 as (name, age, gpa);
D = JOIN B by name, C by name;
store D into ':OUTPATH:';#,
+ 'pig_win' => q#
+define CMD1 `perl -ne "print $_;"`;
+define CMD2 `perl -ne "print $_;"`;
+A = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
+B = stream A through CMD1 as (name, age, gpa);
+store B into ':OUTPATH:.intermediate';
+C = stream B through CMD2 as (name, age, gpa);
+D = JOIN B by name, C by name;
+store D into ':OUTPATH:';#,
'notmq' => 1,
'sql' => "select A.name, A.age, A.gpa, B.name, B.age, B.gpa from studenttab10k as A join studenttab10k as B using(name);",
},
@@ -422,6 +431,14 @@ B = stream A through CMD1;
C = stream B through CMD1;
D = stream C through CMD2;
store D into ':OUTPATH:';#,
+ 'pig_win' => q#
+define CMD1 `perl -ne "print $_;print STDERR "stderr $_";"`;
+define CMD2 `Split.pl 3` input(stdin using PigStreaming(',')) ship(':SCRIPTHOMEPATH:/Split.pl');
+A = load ':INPATH:/singlefile//studenttab10k';
+B = stream A through CMD1;
+C = stream B through CMD1;
+D = stream C through CMD2;
+store D into ':OUTPATH:';#,
'sql' => "select name, age, gpa from studenttab10k;",
},
{
@@ -438,6 +455,16 @@ C = stream B through CMD1;
D = stream C through CMD2;
E = JOIN B by $0, D by $0;
store E into ':OUTPATH:';#,
+ 'pig_win' => q#
+define CMD1 `perl -ne "print $_;"`;
+define CMD2 `Split.pl 3` input(stdin using PigStreaming(',')) ship(':SCRIPTHOMEPATH:/Split.pl');
+A = load ':INPATH:/singlefile/studenttab10k';
+B = stream A through CMD1;
+store B into ':OUTPATH:.intermediate';
+C = stream B through CMD1;
+D = stream C through CMD2;
+E = JOIN B by $0, D by $0;
+store E into ':OUTPATH:';#,
'notmq' => 1,
'sql' => "select A.name, A.age, A.gpa, B.name, B.age, B.gpa from studenttab10k as A join studenttab10k as B using(name);",
},
Modified: pig/branches/branch-0.14/test/e2e/pig/udfs/java/build.xml
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/test/e2e/pig/udfs/java/build.xml?rev=1635710&r1=1635709&r2=1635710&view=diff
==============================================================================
--- pig/branches/branch-0.14/test/e2e/pig/udfs/java/build.xml (original)
+++ pig/branches/branch-0.14/test/e2e/pig/udfs/java/build.xml Fri Oct 31 07:31:17 2014
@@ -21,17 +21,26 @@
<property name="pig.jarfile" value="${lib.dir}/pig.jar" /> -->
<property name="udf.src.dir" value="${basedir}/org/" />
-
<path id="udf-classpath">
<!-- <fileset dir="lib">
<include name="**/*.jar"/>
</fileset>-->
- <fileset dir="${pig.base.dir}/build/ivy/lib/Pig">
+ <fileset dir="${pig.base.dir}/build/ivy/lib/Pig" erroronmissingdir="false">
<include name="*.jar"/>
</fileset>
- <fileset dir="${pig.base.dir}">
+ <fileset dir="${pig.dir}">
<include name="pig*-core-*.jar"/>
</fileset>
+ <fileset dir="${hadoop.common.dir}" erroronmissingdir="false">
+ <include name="hadoop-common*.jar"/>
+ </fileset>
+ <fileset dir="${hadoop.common.lib.dir}" erroronmissingdir="false">
+ <include name="commons-codec*.jar"/>
+ <include name="commons-logging*.jar"/>
+ </fileset>
+ <fileset dir="${hadoop.mapreduce.dir}" erroronmissingdir="false">
+ <include name="hadoop-mapreduce-client-core*.jar"/>
+ </fileset>
</path>
<target name="init">