You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2016/11/03 21:42:54 UTC
svn commit: r1767980 - in /pig/trunk: CHANGES.txt
test/e2e/pig/tests/hcat.conf test/e2e/pig/tests/turing_jython.conf
Author: daijy
Date: Thu Nov 3 21:42:54 2016
New Revision: 1767980
URL: http://svn.apache.org/viewvc?rev=1767980&view=rev
Log:
PIG-5049: Cleanup e2e tests turing_jython.conf
Modified:
pig/trunk/CHANGES.txt
pig/trunk/test/e2e/pig/tests/hcat.conf
pig/trunk/test/e2e/pig/tests/turing_jython.conf
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1767980&r1=1767979&r2=1767980&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Nov 3 21:42:54 2016
@@ -54,6 +54,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-5049: Cleanup e2e tests turing_jython.conf (Daniel Dai)
+
PIG-5033: MultiQueryOptimizerTez creates bad plan with union, split and FRJoin (rohini,tmwoordruff via rohini)
PIG-4934: SET command does not work well with deprecated settings (szita via daijy)
Modified: pig/trunk/test/e2e/pig/tests/hcat.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/hcat.conf?rev=1767980&r1=1767979&r2=1767980&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/hcat.conf (original)
+++ pig/trunk/test/e2e/pig/tests/hcat.conf Thu Nov 3 21:42:54 2016
@@ -55,6 +55,35 @@ store a into ':OUTPATH:';\,
},
]
},
+ {
+ 'name' => 'Jython_HCatDDL',
+ 'tests' => [
+ {
+ # sql command
+ 'num' => 1
+ ,'java_params' => ['-Dhcat.bin=:HCATBIN:']
+ ,'pig' => q\#!/usr/bin/python
+from org.apache.pig.scripting import Pig
+
+#create pig script
+
+Pig.sql("""sql drop table if exists pig_script_hcat_ddl_1;""")
+ret = Pig.sql("""sql create table pig_script_hcat_ddl_1(name string,
+age int,
+gpa double)
+stored as textfile;
+""")
+
+if ret==0:
+ print "SQL command PASSED"
+
+else:
+ raise "SQL command FAILED"
+\
+ ,'rc' => 0
+ },
+ ]
+ },
]
}
;
Modified: pig/trunk/test/e2e/pig/tests/turing_jython.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/turing_jython.conf?rev=1767980&r1=1767979&r2=1767980&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/turing_jython.conf (original)
+++ pig/trunk/test/e2e/pig/tests/turing_jython.conf Thu Nov 3 21:42:54 2016
@@ -92,38 +92,12 @@ hdfs = FileSystem.get(config)
}
]
- },
- {
- 'name' => 'Jython_Embedded',
- 'tests' => [
- {
- 'num' => 1,
- ,'pig' => q\#!/usr/bin/python
-# JYTHON COMMENT
-from org.apache.pig.scripting import Pig
-
-P = Pig.compile("""A = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
-store A into ':OUTPATH:';""")
-
-Q = P.bind()
-
-result = Q.runSingle()
-
-if result.isSuccessful():
- print "Pig job PASSED"
-
-else:
- raise "Pig job FAILED"
-\,
- 'verify_pig_script' => q\A = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
- store A into ':OUTPATH:';\,
- }
- ]
}, {
'name' => 'Jython_CompileBindRun'
,'tests' => [
- { # bind() with no parameters, runSingle
+ {
+ # bind no parameters, runSingle
'num' => 1
,'pig' => q\#!/usr/bin/python
# JYTHON COMMENT
@@ -150,7 +124,7 @@ else:
,'delimiter' => ' '
},{
-# 9.2 1 bind single input parameter and no output parameters
+ # bind single input parameter
'num' => 2
,'pig' => q\#!/usr/bin/python
@@ -179,7 +153,7 @@ else:
# ,'expected_out_regex' => "Pig job PASSED"
},{
-# bind parallel execution with a multiple entries
+ # bind parallel execution with a multiple entries
'num' => 3
,'pig' => q\#!/usr/bin/python
from org.apache.pig.scripting import Pig
@@ -231,9 +205,7 @@ for i in [0, 1, 2]:
\,
},{
-# 8.6 compile pig script file with no input and no output parameters
-#12.2 import python modules
-#
+ # compile pig script file with no parameters
'num' => 4
,'pig' => q\#!/usr/bin/python
from org.apache.pig.scripting import Pig
@@ -243,6 +215,7 @@ pig_script = ":TMP:/script.pig"
pigfile = open( pig_script, 'w+')
pigfile.write("""
A = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
+-- a comment
store A into ':OUTPATH:';
""")
pigfile.close()
@@ -263,7 +236,7 @@ else:
,'floatpostprocess' => 1
,'delimiter' => ' '
},{
-# 8.7 compile pig script file with no input and with output parameters
+ # compile pig script file with parameters
'num' => 5
,'pig' => q\#!/usr/bin/python
from org.apache.pig.scripting import Pig
@@ -300,7 +273,7 @@ else:
,'delimiter' => ' '
},{
- # 11.15 1 results.getResults(alias) for null results
+ # results.getResults(alias) for null results
'num' => 6
,'pig' => q\#!/usr/bin/python
from org.apache.pig.scripting import Pig
@@ -318,7 +291,7 @@ result = P.bind().runSingle()
store EMPTY into ':OUTPATH:';\
},
{
- # bind reading from python context
+ # bind parameters from python context
'num' => 7
,'pig' => q\#!/usr/bin/python
from org.apache.pig.scripting import Pig
@@ -340,7 +313,7 @@ result = P.bind().runSingle()
store B into ':OUTPATH:';\
},{
- # bind multiple times
+ # bind multiple times
'num' => 8
,'pig' => q\#!/usr/bin/python
from org.apache.pig.scripting import Pig
@@ -367,56 +340,8 @@ for i in [1,2,3]:
B= foreach A generate age + 3;
store B into ':OUTPATH:.3';\,
- },
- {
- # invoke .run() on a non-parallel pig script
- 'num' => 9
- ,'pig' => q\#!/usr/bin/python
-from org.apache.pig.scripting import Pig
-
-P = Pig.compile("""
-A = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
-store A into ':OUTPATH:';
-""")
-result = P.bind().run()
-\,
- 'verify_pig_script' => q\A = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
- store A into ':OUTPATH:';\,
- },
- {
-# 8.6 compile pig script file with no input and no output parameters
-#12.2 import python modules
-#
- 'num' => 10
- ,'pig' => q\#!/usr/bin/python
-from org.apache.pig.scripting import Pig
-
-#create pig script
-pig_script = ":TMP:/script.pig"
-pigfile = open( pig_script, 'w+')
-pigfile.write("""
-A = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
--- a comment
-store A into ':OUTPATH:';
-""")
-pigfile.close()
-
-#execute pig script
-
-result = Pig.compileFromFile( pig_script ).bind().runSingle()
-
-if result.isSuccessful():
- print "Pig job PASSED"
-else:
- raise "Pig job FAILED"
-\,
-
- 'verify_pig_script' => q\A = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
- store A into ':OUTPATH:';
-\
- ,'floatpostprocess' => 1
- ,'delimiter' => ' '
},{
+ # python script with parameters
'num' => 11
,'pig_params' => ['-p', qq(loadfile='studenttab10k')],
,'pig' => q\#!/usr/bin/python
@@ -441,6 +366,7 @@ else:
,'floatpostprocess' => 1
,'delimiter' => ' '
},{
+ # python script with parameter file
'num' => 12
,'pig_params' => ['-m', ":PARAMPATH:/params_3"],
,'pig' => q\#!/usr/bin/python
@@ -465,6 +391,7 @@ else:
,'floatpostprocess' => 1
,'delimiter' => ' '
},{
+ # python script with command line arguments
'num' => 13
,'additional_cmd_args' => ['studenttab10k']
,'pig' => q\#!/usr/bin/python
@@ -495,7 +422,7 @@ else:
'name' => 'Jython_Diagnostics'
,'tests' => [
{
-# 11.23 1 explain() on a complex query
+ # explain() on a complex query
'num' => 1
,'pig' => q\#!/usr/bin/python
from org.apache.pig.scripting import Pig
@@ -525,7 +452,7 @@ result = P.bind({'in1':input1, 'in2':inp
,'rc'=> 0
}, {
-#11.22 1 illustrate() on a complex query
+ # illustrate() on a complex query
'num' => 2
,'execonly' => 'mapred,local' #TODO: PIG-3993: Illustrate is yet to be implemented in Tez
,'pig' => q\#!/usr/bin/python
@@ -555,7 +482,7 @@ result = P.bind({'in1':input1, 'in2':inp
,'rc'=> 0
,'expected_out_regex' => "A.*name:bytearray.*age:bytearray.*gpa:bytearray"
}, {
-# 11.24 1 describe() on an alias
+ # describe() on an alias
'num' => 3
,'pig' => q\#!/usr/bin/python
from org.apache.pig.scripting import Pig
@@ -583,7 +510,7 @@ result = P.bind({'in1':input1, 'in2':inp
,'rc'=> 0
,'expected_out_regex' => "A:.*{name:.*bytearray,age:.*bytearray,gpa:.*bytearray}"
}, {
-#11.29 1 describe() on an undefined alias
+ # describe() on an undefined alias
'num' => 4
,'pig' => q\#!/usr/bin/python
from org.apache.pig.scripting import Pig
@@ -613,7 +540,7 @@ result = P.bind({'in1':input1, 'in2':inp
}, {
-# 11.27 1 illustrate(alias)
+ # illustrate(alias)
'num' => 5
,'pig' => q\#!/usr/bin/python
from org.apache.pig.scripting import Pig
@@ -643,7 +570,7 @@ result = P.bind({'in1':input1, 'in2':inp
,'expected_err_regex' => "ERROR 1121"
}, {
-# 11.28 1 explain(alias)
+ # explain(alias)
'num' => 6
,'pig' => q\#!/usr/bin/python
from org.apache.pig.scripting import Pig
@@ -710,14 +637,10 @@ Pig.fs("-copyFromLocal :TMP:/iterator_ou
},
]
}, {
-# 12.2 import python modules
-# 12.1 python comments
-# 12.6 fs lists a file
-
-
'name' => 'Jython_Misc'
,'tests' => [
{
+ # fs commands: lists a file
'num' => 1
,'pig' => q\#!/usr/bin/python
# JYTHON COMMENT
@@ -778,8 +701,8 @@ P.bind().runSingle()
'name' => 'Jython_Properties',
'tests' => [
{
+ # check if property is passed to Pig
'num' => 1
- ,'ignore' => 1 # This is a good test except that we can't verify it.
,'pig' => q\#!/usr/bin/python
# JYTHON COMMENT
from org.apache.pig.scripting import Pig
@@ -791,7 +714,7 @@ store A into ':OUTPATH:';""")
Q = P.bind()
prop = Properties()
-prop.put("mapred.job.name", "friendship")
+prop.put("pig.default.load.func", "wrong")
result = Q.runSingle(prop)
if result.isSuccessful():
@@ -799,10 +722,8 @@ if result.isSuccessful():
else:
raise "Pig job FAILED"
\
-
- ,'sql' => "select name, age, gpa+0.00 from studenttab10k;"
- ,'floatpostprocess' => 1
- ,'delimiter' => ' '
+ ,'rc'=> 6
+ ,'expected_err_regex' => "ERROR 1070: Could not resolve wrong using imports"
}
]
@@ -811,7 +732,7 @@ else:
'name' => 'Jython_Error',
'tests' => [
{
- # run a script that returns single negative result
+ # run a script that returns single negative result
'num' => 1
,'pig' => q\#!/usr/bin/python
from org.apache.pig.scripting import Pig
@@ -834,103 +755,18 @@ else:
,'rc' => 6
,'expected_err_regex' => "ERROR 1121"
- },
- {
- # run a script that returns single negative result
- 'num' => 2
- ,'pig' => q\#!/usr/bin/python
-from org.apache.pig.scripting import Pig
-
-input= ":INPATH:/singlefile/studenttab10k"
-output = ":OUTPATH:"
-
-P = Pig.compile("""A = load '$in' as (name, age, gpa); store A into '$out';""")
-
-Q = P.bind({'in':input, 'out':bad_output})
-
-result = Q.runSingle()
-
-if result.isSuccessful():
- print "Pig job PASSED"
-
-else:
- raise "Pig job FAILED"
-\
-
- ,'rc' => 6
- ,'expected_err_regex' => "name 'bad_output' is not defined"
},{
- # bind an undefined input parameter
- 'num' => 3
- ,'pig' => q\#!/usr/bin/python
-from org.apache.pig.scripting import Pig
-
-input= ":INPATH:/singlefile/studenttab10k"
-output = ":OUTPATH:"
-
-P = Pig.compile("""A = load '$in' as (name, age, gpa); store A into '$out';""")
-
-Q = P.bind({'in':invalid_parameter, 'out':output})
-
-result = Q.runSingle()
-
-if result.isSuccessful():
- print "Pig job PASSED"
-
-else:
- raise "Pig job FAILED"
-\
-
- ,'expected_err_regex' => "ERROR 1121"
- ,'rc'=> 6
-
- },
- {
- # compileFromFile for pig script file that does not exist throws IOException
+ # compileFromFile for pig script file that does not exist throws IOException
'num' => 4
,'pig' => q\#!/usr/bin/python
+import os
from org.apache.pig.scripting import Pig
# intentionally don't create pig script
-pig_script = tmp_dir + "/script.pig"
-
-#execute pig script
-input1= ":INPATH:/singlefile/studenttab10k"
-input2= ":INPATH:/singlefile/votertab10k"
-output1= ":OUTPATH:.1"
-output2= ":OUTPATH:.2"
-
-result = Pig.compileFromFile(pig_script).bind({'in1':input1,'in2':input2, 'out1':output1, 'out2':output2 }).run()
-
-if result.isSuccessful():
- print "Pig job PASSED"
-
-else:
- raise "Pig job FAILED"
-\
-
- ,'expected_err_regex' => "ERROR 1121"
- ,'rc'=> 6
- },
- {
- # compileFromFile for pig script file that does not have read permissions throws IOException
- 'num' => 5
- ,'pig' => q\#!/usr/bin/python
-from org.apache.pig.scripting import Pig
-
-#create pig script
-
pig_script = ":TMP:/script.pig"
-pigfile = open( pig_script, 'w')
-#no read permissions and file is left open until afer compile statement
-pigfile.write("""
-A = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
-B = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
-store A into '$out1';
-store B into '$out2';
-""")
-pigfile.close()
+
+os.remove(pig_script)
#execute pig script
input1= ":INPATH:/singlefile/studenttab10k"
@@ -938,11 +774,9 @@ input2= ":INPATH:/singlefile/votertab10k
output1= ":OUTPATH:.1"
output2= ":OUTPATH:.2"
-result = Pig.compileFromFile(pig_script).bind({'in1':input1,'in2':input2, 'out1':output1, 'out2':output2 }).run()
-
-pigfile.close()
+results = Pig.compileFromFile(pig_script).bind({'in1':input1,'in2':input2, 'out1':output1, 'out2':output2 }).run()
-if result.isSuccessful():
+if results[0].isSuccessful():
print "Pig job PASSED"
else:
@@ -977,13 +811,16 @@ else:
,'expected_err_regex' => "ERROR 1121"
},
{
- # 11.10 iter.next for an alias that is undefined
+ # iter.next for an alias that is undefined
'num' => 7
,'pig' => q\#!/usr/bin/python
from org.apache.pig.scripting import Pig
#create pig script
+out1= ":OUTPATH:.1"
+out2= ":OUTPATH:.2"
+
P = Pig.compile("""A = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
B= filter A by age < 50;
store B into '$out1';
@@ -992,9 +829,10 @@ D = filter C by name matches '^fred*';
store D into '$out2';
""")
-result = P.bind().run()
+results = P.bind().run()
+iter = results[0].result("E").iterator()
-if result.isSuccessful():
+if results[0].isSuccessful():
print "Pig job PASSED"
else:
@@ -1010,30 +848,6 @@ else:
'tests' => [
{
# sql command
- 'num' => 1
- ,'java_params' => ['-Dhcat.bin=:HCATBIN:']
- ,'pig' => q\#!/usr/bin/python
-from org.apache.pig.scripting import Pig
-
-#create pig script
-
-Pig.sql("""sql drop table if exists pig_script_hcat_ddl_1;""")
-ret = Pig.sql("""sql create table pig_script_hcat_ddl_1(name string,
-age int,
-gpa double)
-stored as textfile;
-""")
-
-if ret==0:
- print "SQL command PASSED"
-
-else:
- raise "SQL command FAILED"
-\
- ,'rc' => 0
- },
- {
- # sql command
'num' => 2
,'pig' => q\#!/usr/bin/python
from org.apache.pig.scripting import Pig