You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2013/08/14 19:07:07 UTC
svn commit: r1513967 - in /pig/trunk: ./ src/org/apache/pig/parser/
test/org/apache/pig/data/ test/org/apache/pig/test/
Author: cheolsoo
Date: Wed Aug 14 17:07:06 2013
New Revision: 1513967
URL: http://svn.apache.org/r1513967
Log:
PIG-3414: QueryParserDriver.parseSchema(String) silently returns a wrong result when a comma is missing in the schema definition (cheolsoo)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/parser/QueryParser.g
pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
pig/trunk/test/org/apache/pig/data/TestSchemaTuple.java
pig/trunk/test/org/apache/pig/test/TestPOCast.java
pig/trunk/test/org/apache/pig/test/TestSchema.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1513967&r1=1513966&r2=1513967&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Aug 14 17:07:06 2013
@@ -210,6 +210,8 @@ PIG-3013: BinInterSedes improve chararra
BUG FIXES
+PIG-3414: QueryParserDriver.parseSchema(String) silently returns a wrong result when a comma is missing in the schema definition (cheolsoo)
+
PIG-3412: jsonstorage breaks when tuple does not have as many columns as schema (aesilberstein via cheolsoo)
PIG-3243: Documentation error (sarutak via cheolsoo)
Modified: pig/trunk/src/org/apache/pig/parser/QueryParser.g
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParser.g?rev=1513967&r1=1513966&r2=1513967&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryParser.g (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParser.g Wed Aug 14 17:07:06 2013
@@ -208,6 +208,9 @@ catch(RecognitionException re) {
query : statement* EOF -> ^( QUERY statement* )
;
+schema: field_def_list EOF
+;
+
// STATEMENTS
statement : SEMI_COLON!
Modified: pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java?rev=1513967&r1=1513966&r2=1513967&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java Wed Aug 14 17:07:06 2013
@@ -50,8 +50,8 @@ import org.apache.pig.impl.io.ResourceNo
import org.apache.pig.newplan.Operator;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
import org.apache.pig.newplan.logical.relational.LogicalSchema;
-import org.apache.pig.parser.QueryParser.field_def_list_return;
import org.apache.pig.parser.QueryParser.literal_return;
+import org.apache.pig.parser.QueryParser.schema_return;
import org.apache.pig.tools.pigstats.ScriptState;
public class QueryParserDriver {
@@ -85,9 +85,9 @@ public class QueryParserDriver {
private static Tree parseSchema(CommonTokenStream tokens) throws ParserException {
QueryParser parser = QueryParserUtils.createParser(tokens);
- field_def_list_return result = null;
+ schema_return result = null;
try {
- result = parser.field_def_list();
+ result = parser.schema();
} catch (RecognitionException e) {
String msg = parser.getErrorHeader(e) + " "
+ parser.getErrorMessage(e, parser.getTokenNames());
Modified: pig/trunk/test/org/apache/pig/data/TestSchemaTuple.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/data/TestSchemaTuple.java?rev=1513967&r1=1513966&r2=1513967&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/data/TestSchemaTuple.java (original)
+++ pig/trunk/test/org/apache/pig/data/TestSchemaTuple.java Wed Aug 14 17:07:06 2013
@@ -131,40 +131,40 @@ public class TestSchemaTuple {
udfSchema = Utils.getSchemaFromString(
"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))");
@@ -260,40 +260,40 @@ public class TestSchemaTuple {
udfSchema = Utils.getSchemaFromString(
"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
- +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))"
+ +"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger)),"
+"int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger,"
+"(int, long, chararray, boolean, bytearray, float, double, datetime, bigdecimal, biginteger))");
Modified: pig/trunk/test/org/apache/pig/test/TestPOCast.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPOCast.java?rev=1513967&r1=1513966&r2=1513967&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPOCast.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPOCast.java Wed Aug 14 17:07:06 2013
@@ -2174,7 +2174,7 @@ public class TestPOCast {
wrappedTuple.append(GenRandomData.genRandString(r));
wrappedTuple.append(GenRandomData.genRandString(r));
t.append(wrappedTuple);
- Schema s = Utils.getSchemaFromString("t:tuple(a:chararray)}");
+ Schema s = Utils.getSchemaFromString("t:tuple(a:chararray)");
op.setFieldSchema(new ResourceSchema.ResourceFieldSchema(s.getField(0)));
plan.attachInput(t);
Tuple tup = null;
Modified: pig/trunk/test/org/apache/pig/test/TestSchema.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestSchema.java?rev=1513967&r1=1513966&r2=1513967&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestSchema.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestSchema.java Wed Aug 14 17:07:06 2013
@@ -868,7 +868,7 @@ public class TestSchema {
"datetime,datetime,datetime,datetime,datetime,datetime,datetime,datetime,datetime,datetime",
"{},{},{},{},{},{},{},{},{},{}",
"map[],map[],map[],map[],map[],map[],map[],map[],map[],map[]",
- "int,int,long,long,float,float,double,double,boolean,boolean,datetime,datetime(int,long,float,double,boolean,datetime),{(int,long,float,double,boolean,datetime)},map[(int,long,float,double,boolean,datetime)]"
+ "int,int,long,long,float,float,double,double,boolean,boolean,datetime,datetime,(int,long,float,double,boolean,datetime),{(int,long,float,double,boolean,datetime)},map[(int,long,float,double,boolean,datetime)]"
};
for (String schemaString : schemaStrings) {
Schema s1 = Utils.getSchemaFromString(schemaString);
@@ -877,4 +877,11 @@ public class TestSchema {
assertTrue(Schema.equals(s1,s2,false,true));
}
}
-}
\ No newline at end of file
+
+ @Test(expected=ParserException.class)
+ public void testGetStringFromSchemaNegative() throws Exception {
+ String schemaString = "a:int b:long"; // A comma is missing between fields
+ Utils.getSchemaFromString(schemaString);
+ fail("The schema string is invalid, so parsing should fail!");
+ }
+}