You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2012/01/09 07:48:10 UTC
svn commit: r1229041 - in /pig/trunk: ./ src/org/apache/pig/impl/util/
src/org/apache/pig/parser/ test/org/apache/pig/parser/
Author: daijy
Date: Mon Jan 9 06:48:09 2012
New Revision: 1229041
URL: http://svn.apache.org/viewvc?rev=1229041&view=rev
Log:
PIG-2267: Make the name of the columns in schema optional
Added:
pig/trunk/src/org/apache/pig/impl/util/NumValCarrier.java
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/parser/AliasMasker.g
pig/trunk/src/org/apache/pig/parser/AstPrinter.g
pig/trunk/src/org/apache/pig/parser/AstValidator.g
pig/trunk/src/org/apache/pig/parser/LogicalPlanGenerator.g
pig/trunk/src/org/apache/pig/parser/QueryParser.g
pig/trunk/test/org/apache/pig/parser/TestErrorHandling.java
pig/trunk/test/org/apache/pig/parser/TestQueryParser.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1229041&r1=1229040&r2=1229041&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Jan 9 06:48:09 2012
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-2267: Make the name of the columns in schema optional (jcoveney via daijy)
+
PIG-2453: Fetching schema can be very slow for multi-thousand file LOADs (dvryaboy)
PIG-2443: [Piggybank] Add UDFs to check if a String is an Integer And if a String is Numeric (prkommireddi via daijy)
Added: pig/trunk/src/org/apache/pig/impl/util/NumValCarrier.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/NumValCarrier.java?rev=1229041&view=auto
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/util/NumValCarrier.java (added)
+++ pig/trunk/src/org/apache/pig/impl/util/NumValCarrier.java Mon Jan 9 06:48:09 2012
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.impl.util;
+
+import java.util.Map;
+import java.util.HashMap;
+
+import org.apache.pig.data.DataType;
+
+public class NumValCarrier {
+ private Map<Byte,ValCarrier> byteToStr = new HashMap<Byte,ValCarrier>(10);
+
+ public NumValCarrier() {
+ ValCarrier valCarrier = new ValCarrier("val");
+ ValCarrier bagCarrier = new ValCarrier("bag");
+ ValCarrier tupleCarrier = new ValCarrier("tuple");
+ ValCarrier mapCarrier = new ValCarrier("map");
+ ValCarrier nullCarrier = new ValCarrier("null");
+
+ byteToStr.put(DataType.BAG,bagCarrier);
+ byteToStr.put(DataType.CHARARRAY,valCarrier);
+ byteToStr.put(DataType.BYTEARRAY,valCarrier);
+ byteToStr.put(DataType.DOUBLE,valCarrier);
+ byteToStr.put(DataType.FLOAT,valCarrier);
+ byteToStr.put(DataType.INTEGER,valCarrier);
+ byteToStr.put(DataType.LONG,valCarrier);
+ byteToStr.put(DataType.MAP,mapCarrier);
+ byteToStr.put(DataType.TUPLE,tupleCarrier);
+ byteToStr.put(DataType.NULL,nullCarrier);
+ }
+
+ public String makeNameFromDataType(byte type) {
+ return byteToStr.get(type).getNextString();
+ }
+
+ private static class ValCarrier {
+ private int num=0;
+ private String str;
+
+ public ValCarrier(String str) {
+ this.str=str;
+ }
+
+ public String getNextString() {
+ return str+"_"+num++;
+ }
+ }
+}
Modified: pig/trunk/src/org/apache/pig/parser/AliasMasker.g
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/AliasMasker.g?rev=1229041&r1=1229040&r2=1229041&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/AliasMasker.g (original)
+++ pig/trunk/src/org/apache/pig/parser/AliasMasker.g Mon Jan 9 06:48:09 2012
@@ -191,6 +191,7 @@ field_def
}
}
}
+ | ^( FIELD_DEF_WITHOUT_IDENTIFIER type )
;
field_def_list
Modified: pig/trunk/src/org/apache/pig/parser/AstPrinter.g
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/AstPrinter.g?rev=1229041&r1=1229040&r2=1229041&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/AstPrinter.g (original)
+++ pig/trunk/src/org/apache/pig/parser/AstPrinter.g Mon Jan 9 06:48:09 2012
@@ -151,6 +151,7 @@ as_clause
field_def
: ^( FIELD_DEF IDENTIFIER { sb.append($IDENTIFIER.text); } ( {sb.append(":"); } type)? )
+ | ^( FIELD_DEF type )
;
field_def_list
Modified: pig/trunk/src/org/apache/pig/parser/AstValidator.g
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/AstValidator.g?rev=1229041&r1=1229040&r2=1229041&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/AstValidator.g (original)
+++ pig/trunk/src/org/apache/pig/parser/AstValidator.g Mon Jan 9 06:48:09 2012
@@ -35,6 +35,9 @@ options {
@header {
package org.apache.pig.parser;
+import org.apache.pig.data.DataType;
+import org.apache.pig.impl.util.NumValCarrier;
+
import java.util.HashSet;
import java.util.Set;
@@ -186,24 +189,38 @@ filename : QUOTEDSTRING
as_clause: ^( AS field_def_list )
;
-field_def[Set<String> fieldNames] throws DuplicatedSchemaAliasException
+field_def[Set<String> fieldNames, NumValCarrier nvc] throws DuplicatedSchemaAliasException
: ^( FIELD_DEF IDENTIFIER { validateSchemaAliasName( fieldNames, $IDENTIFIER, $IDENTIFIER.text ); } type? )
+ | ^( FIELD_DEF_WITHOUT_IDENTIFIER type { validateSchemaAliasName ( fieldNames, $FIELD_DEF_WITHOUT_IDENTIFIER, $nvc.makeNameFromDataType ( $type.typev ) ); } )
;
field_def_list throws DuplicatedSchemaAliasException
scope{
Set<String> fieldNames;
+ NumValCarrier nvc;
}
@init {
$field_def_list::fieldNames = new HashSet<String>();
+ $field_def_list::nvc = new NumValCarrier();
}
- : ( field_def[$field_def_list::fieldNames] )+
-;
-
-type : simple_type | tuple_type | bag_type | map_type
+ : ( field_def[$field_def_list::fieldNames, $field_def_list::nvc] )+
;
-simple_type : BOOLEAN | INT | LONG | FLOAT | DOUBLE | CHARARRAY | BYTEARRAY
+type returns [byte typev]
+ : simple_type { $typev = $simple_type.typev; }
+ | tuple_type { $typev = DataType.TUPLE; }
+ | bag_type { $typev = DataType.BAG; }
+ | map_type { $typev = DataType.MAP; }
+;
+
+simple_type returns [byte typev]
+ : BOOLEAN { $typev = DataType.BOOLEAN; }
+ | INT { $typev = DataType.INTEGER; }
+ | LONG { $typev = DataType.LONG; }
+ | FLOAT { $typev = DataType.FLOAT; }
+ | DOUBLE { $typev = DataType.DOUBLE; }
+ | CHARARRAY { $typev = DataType.CHARARRAY; }
+ | BYTEARRAY { $typev = DataType.BYTEARRAY; }
;
tuple_type : ^( TUPLE_TYPE field_def_list? )
Modified: pig/trunk/src/org/apache/pig/parser/LogicalPlanGenerator.g
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/LogicalPlanGenerator.g?rev=1229041&r1=1229040&r2=1229041&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/LogicalPlanGenerator.g (original)
+++ pig/trunk/src/org/apache/pig/parser/LogicalPlanGenerator.g Mon Jan 9 06:48:09 2012
@@ -47,6 +47,7 @@ import org.apache.pig.impl.logicalLayer.
import org.apache.pig.impl.streaming.StreamingCommand;
import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec;
import org.apache.pig.impl.util.MultiMap;
+import org.apache.pig.impl.util.NumValCarrier;
import org.apache.pig.impl.plan.PlanValidationException;
import org.apache.pig.newplan.Operator;
import org.apache.pig.newplan.logical.expression.AddExpression;
@@ -313,21 +314,29 @@ as_clause returns[LogicalSchema logicalS
}
;
-field_def returns[LogicalFieldSchema fieldSchema]
+field_def[NumValCarrier nvc] returns[LogicalFieldSchema fieldSchema]
@init {
byte datatype = DataType.NULL;
+ if ($nvc==null) {
+ $nvc=new NumValCarrier();
+ }
}
: ^( FIELD_DEF IDENTIFIER ( type { datatype = $type.datatype;} )? )
{
$fieldSchema = new LogicalFieldSchema( $IDENTIFIER.text, $type.logicalSchema, datatype );
}
+ | ^( FIELD_DEF_WITHOUT_IDENTIFIER ( type { datatype = $type.datatype; } ) )
+ {
+ $fieldSchema = new LogicalFieldSchema ( $nvc.makeNameFromDataType(datatype) , $type.logicalSchema, datatype );
+ }
;
field_def_list returns[LogicalSchema schema]
@init {
$schema = new LogicalSchema();
+ NumValCarrier nvc = new NumValCarrier();
}
- : ( field_def { $schema.addField( $field_def.fieldSchema ); } )+
+ : ( field_def[nvc] { $schema.addField( $field_def.fieldSchema ); } )+
;
Modified: pig/trunk/src/org/apache/pig/parser/QueryParser.g
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParser.g?rev=1229041&r1=1229040&r2=1229041&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryParser.g (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParser.g Mon Jan 9 06:48:09 2012
@@ -47,6 +47,7 @@ tokens {
BAG_VAL;
KEY_VAL_PAIR;
FIELD_DEF;
+ FIELD_DEF_WITHOUT_IDENTIFIER;
NESTED_CMD_ASSI;
NESTED_CMD;
NESTED_PROJ;
@@ -291,11 +292,13 @@ load_clause : LOAD^ filename ( USING! fu
filename : QUOTEDSTRING
;
-as_clause: AS^ ( field_def | ( LEFT_PAREN! field_def_list RIGHT_PAREN! ) )
+as_clause: AS^ ( ( LEFT_PAREN! field_def_list RIGHT_PAREN! ) | field_def )
;
field_def : identifier ( COLON type )?
-> ^( FIELD_DEF identifier type? )
+ | type
+ -> ^( FIELD_DEF_WITHOUT_IDENTIFIER type )
;
field_def_list : field_def ( COMMA field_def )*
@@ -353,10 +356,10 @@ rel : alias
| LEFT_PAREN! ( foreach_clause_complex | ( ( op_clause | foreach_clause_simple ) parallel_clause? ) ) RIGHT_PAREN!
;
-flatten_generated_item : flatten_clause ( AS! ( field_def | ( LEFT_PAREN! field_def_list RIGHT_PAREN! ) ) )?
- | col_range ( AS! ( field_def | ( LEFT_PAREN! field_def_list RIGHT_PAREN! ) ) )?
+flatten_generated_item : flatten_clause ( AS! ( ( LEFT_PAREN! field_def_list RIGHT_PAREN! ) | field_def ) )?
+ | col_range ( AS! ( ( LEFT_PAREN! field_def_list RIGHT_PAREN! ) | field_def ) )?
| expr ( AS! field_def )?
- | STAR ( AS! ( field_def | ( LEFT_PAREN! field_def_list RIGHT_PAREN! ) ) )?
+ | STAR ( AS! ( ( LEFT_PAREN! field_def_list RIGHT_PAREN! ) | field_def ) )?
;
flatten_clause : FLATTEN^ LEFT_PAREN! expr RIGHT_PAREN!
Modified: pig/trunk/test/org/apache/pig/parser/TestErrorHandling.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/parser/TestErrorHandling.java?rev=1229041&r1=1229040&r2=1229041&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/parser/TestErrorHandling.java (original)
+++ pig/trunk/test/org/apache/pig/parser/TestErrorHandling.java Mon Jan 9 06:48:09 2012
@@ -241,4 +241,60 @@ public class TestErrorHandling {
Assert.fail( "Testcase should fail" );
}
+ @Test //pig-2267
+ public void testAutomaticallyGivenSchemaName1() throws IOException{
+ String query = "a = load 'x' as (int,val_0);";
+ String failMsg = "Duplicated alias in schema";
+ shouldFailWithMessage(query,failMsg);
+ }
+
+ @Test //pig-2267
+ public void testAutomaticallyGivenSchemaName2() throws IOException{
+ String query = "a = load 'x' as ((int,int),tuple_0:tuple(int,int));";
+ String failMsg = "Duplicated alias in schema";
+ shouldFailWithMessage(query,failMsg);
+ }
+
+ @Test //pig-2267
+ public void testAutomaticallyGivenSchemaName3() throws IOException{
+ String query = "a = load 'x' as (tuple_0:tuple(int,int),(int,int));";
+ String failMsg = "Duplicated alias in schema";
+ shouldFailWithMessage(query,failMsg);
+ }
+
+ @Test //pig-2267
+ public void testAutomaticallyGivenSchemaName4() throws IOException{
+ String query = "a = load 'x' as (bag_0:bag{},{});";
+ String failMsg = "Duplicated alias in schema";
+ shouldFailWithMessage(query,failMsg);
+ }
+
+ @Test //pig-2267
+ public void testAutomaticallyGivenSchemaName5() throws IOException{
+ String query = "a = load 'x' as ([],map_0:map[]);";
+ String failMsg = "Duplicated alias in schema";
+ shouldFailWithMessage(query,failMsg);
+ }
+
+ @Test //pig-2267
+ public void testAutomaticallyGivenSchemaName6() throws IOException{
+ String query = "a = load 'x' as (int,int,int,val_3,val_2);";
+ String failMsg = "Duplicated alias in schema";
+ shouldFailWithMessage(query,failMsg);
+ }
+
+ public void shouldFailWithMessage(String query, String... messages) throws IOException {
+ try {
+ pig.registerQuery(query);
+ } catch (Exception e) {
+ String msg = e.getMessage();
+ System.out.println(msg);
+ for (String message : messages) {
+ Assert.assertTrue(msg.contains(message));
+ }
+ return;
+ }
+ Assert.fail("Testcase should fail");
+ }
+
}
Modified: pig/trunk/test/org/apache/pig/parser/TestQueryParser.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/parser/TestQueryParser.java?rev=1229041&r1=1229040&r2=1229041&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/parser/TestQueryParser.java (original)
+++ pig/trunk/test/org/apache/pig/parser/TestQueryParser.java Mon Jan 9 06:48:09 2012
@@ -331,4 +331,13 @@ public class TestQueryParser {
return parser.getNumberOfSyntaxErrors();
}
+ //PIG-2267
+ public void testThatColNameIsGeneratedProperly() throws IOException {
+ String query = "a = load '1.txt' as (int,(long,[]),{([])});"
+ + "b = foreach a generate val_0, tuple_0, bag_0;"
+ + "c = foreach b generate val_0, flatten(tuple_0), flatten(bag_0);"
+ + "d = foreach c generate val_0, tuple_0::val_0, bag_0::map_0;";
+ PigServer pig = new PigServer(ExecType.LOCAL);
+ Util.registerMultiLineQuery(pig, query);
+ }
}