You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by gd...@apache.org on 2011/10/25 16:28:13 UTC
svn commit: r1188667 - in /pig/trunk: ./ src/org/apache/pig/parser/
test/e2e/pig/tests/
Author: gdfm
Date: Tue Oct 25 14:28:13 2011
New Revision: 1188667
URL: http://svn.apache.org/viewvc?rev=1188667&view=rev
Log:
PIG-1387: Syntactical Sugar for PIG-1385 (azaroth)
Syntactic sugar to convert to tuples "()", bags "{}" and maps "[]".
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/parser/AliasMasker.g
pig/trunk/src/org/apache/pig/parser/AstPrinter.g
pig/trunk/src/org/apache/pig/parser/AstValidator.g
pig/trunk/src/org/apache/pig/parser/LogicalPlanGenerator.g
pig/trunk/src/org/apache/pig/parser/QueryLexer.g
pig/trunk/src/org/apache/pig/parser/QueryParser.g
pig/trunk/test/e2e/pig/tests/nightly.conf
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1188667&r1=1188666&r2=1188667&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Oct 25 14:28:13 2011
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-1387: Syntactical Sugar for PIG-1385 (azaroth)
+
PIG-2305: Pig should log the split locations in task logs (vivekp via thejas)
PIG-2293: Pig should support a more efficient merge join against data sources that natively support point
Modified: pig/trunk/src/org/apache/pig/parser/AliasMasker.g
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/AliasMasker.g?rev=1188667&r1=1188666&r2=1188667&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/AliasMasker.g (original)
+++ pig/trunk/src/org/apache/pig/parser/AliasMasker.g Tue Oct 25 14:28:13 2011
@@ -619,7 +619,10 @@ eid : rel_str_op
| LEFT
| RIGHT
| FULL
- | IDENTIFIER
+ | IDENTIFIER
+ | TOBAG
+ | TOMAP
+ | TOTUPLE
;
// relational operator
Modified: pig/trunk/src/org/apache/pig/parser/AstPrinter.g
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/AstPrinter.g?rev=1188667&r1=1188666&r2=1188667&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/AstPrinter.g (original)
+++ pig/trunk/src/org/apache/pig/parser/AstPrinter.g Tue Oct 25 14:28:13 2011
@@ -602,6 +602,9 @@ eid : rel_str_op
| RIGHT { sb.append($RIGHT.text); }
| FULL { sb.append($FULL.text); }
| IDENTIFIER { sb.append($IDENTIFIER.text); }
+ | TOBAG { sb.append("TOBAG"); }
+ | TOMAP { sb.append("TOMAP"); }
+ | TOTUPLE { sb.append("TOTUPLE"); }
;
// relational operator
Modified: pig/trunk/src/org/apache/pig/parser/AstValidator.g
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/AstValidator.g?rev=1188667&r1=1188666&r2=1188667&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/AstValidator.g (original)
+++ pig/trunk/src/org/apache/pig/parser/AstValidator.g Tue Oct 25 14:28:13 2011
@@ -258,7 +258,7 @@ rel : alias { validateAliasRef( aliases
| op_clause parallel_clause?
;
-flatten_generated_item : ( flatten_clause | col_range | expr | STAR) field_def_list?
+flatten_generated_item : ( flatten_clause | col_range | expr | STAR ) field_def_list?
;
flatten_clause : ^( FLATTEN expr )
@@ -595,6 +595,9 @@ eid : rel_str_op
| RIGHT
| FULL
| IDENTIFIER
+ | TOBAG
+ | TOMAP
+ | TOTUPLE
;
// relational operator
Modified: pig/trunk/src/org/apache/pig/parser/LogicalPlanGenerator.g
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/LogicalPlanGenerator.g?rev=1188667&r1=1188666&r2=1188667&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/LogicalPlanGenerator.g (original)
+++ pig/trunk/src/org/apache/pig/parser/LogicalPlanGenerator.g Tue Oct 25 14:28:13 2011
@@ -1628,6 +1628,9 @@ eid returns[String id] : rel_str_op { $i
| RIGHT { $id = $RIGHT.text; }
| FULL { $id = $FULL.text; }
| IDENTIFIER { $id = $IDENTIFIER.text; }
+ | TOBAG { $id = "TOBAG"; }
+ | TOMAP { $id = "TOMAP"; }
+ | TOTUPLE { $id = "TOTUPLE"; }
;
// relational operator
Modified: pig/trunk/src/org/apache/pig/parser/QueryLexer.g
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryLexer.g?rev=1188667&r1=1188666&r2=1188667&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryLexer.g (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryLexer.g Tue Oct 25 14:28:13 2011
@@ -413,4 +413,3 @@ MINUS : '-'
QMARK : '?'
;
-
Modified: pig/trunk/src/org/apache/pig/parser/QueryParser.g
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParser.g?rev=1188667&r1=1188666&r2=1188667&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryParser.g (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParser.g Tue Oct 25 14:28:13 2011
@@ -1,3 +1,4 @@
+
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -70,6 +71,9 @@ tokens {
FALSE;
IDENTIFIER;
ANY;
+ TOBAG;
+ TOMAP;
+ TOTUPLE;
}
@header {
@@ -353,7 +357,7 @@ flatten_generated_item : flatten_clause
| expr ( AS! field_def )?
| STAR ( AS! ( field_def | ( LEFT_PAREN! field_def_list RIGHT_PAREN! ) ) )?
;
-
+
flatten_clause : FLATTEN^ LEFT_PAREN! expr RIGHT_PAREN!
;
@@ -433,7 +437,15 @@ expr_eval : const_expr | var_expr
var_expr : projectable_expr ( dot_proj | pound_proj )*
;
-projectable_expr: func_eval | col_ref | bin_expr
+projectable_expr: func_eval | col_ref | bin_expr | type_conversion
+;
+
+type_conversion : LEFT_CURLY real_arg_list RIGHT_CURLY
+ -> ^( FUNC_EVAL TOBAG real_arg_list )
+ | LEFT_BRACKET real_arg_list RIGHT_BRACKET
+ -> ^( FUNC_EVAL TOMAP real_arg_list )
+ | LEFT_PAREN real_arg ( COMMA real_arg )+ RIGHT_PAREN // to disable convertion on 1 element tuples
+ -> ^( FUNC_EVAL TOTUPLE real_arg+ )
;
dot_proj : PERIOD ( col_alias_or_index
Modified: pig/trunk/test/e2e/pig/tests/nightly.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/nightly.conf?rev=1188667&r1=1188666&r2=1188667&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/nightly.conf (original)
+++ pig/trunk/test/e2e/pig/tests/nightly.conf Tue Oct 25 14:28:13 2011
@@ -3457,7 +3457,7 @@ store R4 into ':OUTPATH:.6';
?
}, {
-# TESST more TOTUPLE and TOBAG nested combinations
+# TEST more TOTUPLE and TOBAG nested combinations
'num' => 8
,'pig' => q?
A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double);
@@ -3500,6 +3500,56 @@ C = foreach B generate $0, $1, TOBAG($5,
},{
+ 'name' => 'ToStuffSyntaxSugar',
+ 'tests' => [
+ {
+ #TEST TOTUPLE syntax sugar
+ 'num' => 1,
+ 'pig' => q\
+ A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
+ B = foreach A generate (name, age);
+ store B into ':OUTPATH:';\,
+ 'verify_pig_script' => q\
+ A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
+ B = foreach A generate TOTUPLE(name, age);
+ store B into ':OUTPATH:';\,
+ }, {
+ #TEST TOBAG syntax sugar
+ 'num' => 2,
+ 'pig' => q\
+ A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
+ B = foreach A generate {name, age};
+ store B into ':OUTPATH:';\,
+ 'verify_pig_script' => q\
+ A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
+ B = foreach A generate TOBAG(name, age);
+ store B into ':OUTPATH:';\,
+ }, {
+ #TEST TOMAP syntax sugar
+ 'num' => 3,
+ 'pig' => q\
+ A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
+ B = foreach A generate [name, age];
+ store B into ':OUTPATH:';\,
+ 'verify_pig_script' => q\
+ A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
+ B = foreach A generate TOMAP(name, age);
+ store B into ':OUTPATH:';\,
+ }, {
+ #TEST verify single element inside parenthesis does NOT call TOTUPLE
+ 'num' => 4,
+ 'pig' => q\
+ A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
+ B = foreach A generate (age) + 1;
+ store B into ':OUTPATH:';\,
+ 'verify_pig_script' => q\
+ A = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:float);
+ B = foreach A generate (age + 1);
+ store B into ':OUTPATH:';\,
+ }
+ ] # end of tests
+ },{
+
'name' => 'MergeOperator',
#'ignore' => 1, # Need to investigate failures
'tests' => [