You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2016/05/31 19:49:00 UTC
svn commit: r1746334 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/newplan/logical/relational/LogicalSchema.java
src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java
test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java
Author: daijy
Date: Tue May 31 19:49:00 2016
New Revision: 1746334
URL: http://svn.apache.org/viewvc?rev=1746334&view=rev
Log:
PIG-4734: TOMAP schema inferring breaks some scripts in type checking for bincond
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalSchema.java
pig/trunk/src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java
pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1746334&r1=1746333&r2=1746334&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue May 31 19:49:00 2016
@@ -141,6 +141,8 @@ PIG-4639: Add better parser for Apache H
BUG FIXES
+PIG-4734: TOMAP schema inferring breaks some scripts in type checking for bincond (daijy)
+
PIG-4786: CROSS will not work correctly with Grace Parallelism (daijy)
PIG-3227: SearchEngineExtractor does not work for bing (dannyant via daijy)
Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalSchema.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalSchema.java?rev=1746334&r1=1746333&r2=1746334&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalSchema.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalSchema.java Tue May 31 19:49:00 2016
@@ -447,7 +447,23 @@ public class LogicalSchema {
LogicalFieldSchema mergedFS = new LogicalFieldSchema(mergedAlias, mergedSubSchema, mergedType);
return mergedFS;
}
-
+
+ public static boolean isEqualUnlessUnknown(LogicalFieldSchema fs1, LogicalFieldSchema fs2) throws FrontendException {
+ if (fs1.type == DataType.BYTEARRAY) {
+ return true;
+ } else if (fs2.type == DataType.BYTEARRAY) {
+ return true;
+ } else if (fs1.type == fs2.type) {
+ if (DataType.isComplex(fs1.type)) {
+ return LogicalSchema.isEqualUnlessUnknown(fs1.schema, fs2.schema);
+ } else {
+ return true;
+ }
+ } else {
+ return false;
+ }
+ }
+
/***
* Old Pig field schema does not require a tuple schema inside a bag;
* Now it is required to have that; this method is to fill the gap
@@ -770,7 +786,24 @@ public class LogicalSchema {
}
return mergedSchema;
}
-
+
+ public static boolean isEqualUnlessUnknown(LogicalSchema s1, LogicalSchema s2) throws FrontendException {
+ if (s1 == null) {
+ return true;
+ } else if (s2 == null) {
+ return true;
+ } else if (s1.size() != s2.size()) {
+ return false;
+ } else {
+ for (int i=0;i<s1.size();i++) {
+ if (!LogicalFieldSchema.isEqualUnlessUnknown(s1.getField(i), s1.getField(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+ }
+
public String toString(boolean verbose) {
StringBuilder str = new StringBuilder();
Modified: pig/trunk/src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java?rev=1746334&r1=1746333&r2=1746334&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/visitor/TypeCheckingExpVisitor.java Tue May 31 19:49:00 2016
@@ -607,7 +607,7 @@ public class TypeCheckingExpVisitor exte
// Matching schemas if we're working with tuples/bags
if (DataType.isSchemaType(lhsType)) {
try {
- if(! binCond.getLhs().getFieldSchema().isEqual(binCond.getRhs().getFieldSchema())){
+ if(!LogicalFieldSchema.isEqualUnlessUnknown(binCond.getLhs().getFieldSchema(), binCond.getRhs().getFieldSchema())){
int errCode = 1048;
String msg = "Two inputs of BinCond must have compatible schemas."
+ " left hand side: " + binCond.getLhs().getFieldSchema()
Modified: pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java?rev=1746334&r1=1746333&r2=1746334&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java Tue May 31 19:49:00 2016
@@ -39,8 +39,10 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import junit.framework.Assert;
@@ -55,6 +57,8 @@ import org.apache.pig.builtin.PigStorage
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.DataType;
+import org.apache.pig.data.DefaultTuple;
+import org.apache.pig.data.NonSpillableDataBag;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.PigContext;
@@ -4118,4 +4122,56 @@ public class TestTypeCheckingValidatorNe
" corresponding column in earlier relation(s) in the statement";
Util.checkExceptionMessage(query, "c", msg);
}
+ //see PIG-4734
+ public static class GenericToMap extends EvalFunc<Map<String, Double>> {
+ @Override
+ public Map exec(Tuple input) throws IOException {
+ Map<String, Double> output = new HashMap<String, Double>();
+ output.put((String)input.get(0), (Double)input.get(1));
+ return output;
+ }
+ }
+ @Test
+ public void testBinCondCompatMap() throws Exception {
+ String query =
+ "a = load 'studenttab10k' as (name:chararray, gpa:double);"
+ + "b = foreach a generate gpa, TOMAP(name, gpa) as m1, "
+ + GenericToMap.class.getName() + "(name, gpa) as m2;"
+ + "c = foreach b generate (gpa>3? m1 : m2);";
+ createAndProcessLPlan(query);
+ }
+ public static class GenericToTuple extends EvalFunc<Tuple> {
+ @Override
+ public Tuple exec(Tuple input) throws IOException {
+ return input;
+ }
+ }
+ @Test
+ public void testBinCondCompatTuple() throws Exception {
+ String query =
+ "a = load 'studenttab10k' as (name:chararray, gpa:double);"
+ + "b = foreach a generate gpa, TOTUPLE(name, gpa) as t1, "
+ + GenericToTuple.class.getName() + "(name, gpa) as t2;"
+ + "c = foreach b generate (gpa>3? t1 : t2);";
+ createAndProcessLPlan(query);
+ }
+ public static class GenericToBag extends EvalFunc<DataBag> {
+ @Override
+ public DataBag exec(Tuple input) throws IOException {
+ DataBag bag = new NonSpillableDataBag(1);
+ Tuple t = new DefaultTuple();
+ t.append(input.get(0));
+ bag.add(t);
+ return bag;
+ }
+ }
+ @Test
+ public void testBinCondCompatBag() throws Exception {
+ String query =
+ "a = load 'studenttab10k' as (name:chararray, gpa:double);"
+ + "b = foreach a generate gpa, TOBAG(name) as b1, "
+ + GenericToBag.class.getName() + "(name) as b2;"
+ + "c = foreach b generate (gpa>3? b1 : b2);";
+ createAndProcessLPlan(query);
+ }
}