You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by kn...@apache.org on 2016/09/08 19:41:55 UTC
svn commit: r1759894 - in /pig/branches/branch-0.16: CHANGES.txt
src/org/apache/pig/newplan/logical/visitor/LineageFindRelVisitor.java
test/org/apache/pig/test/TestLineageFindRelVisitor.java
Author: knoguchi
Date: Thu Sep 8 19:41:55 2016
New Revision: 1759894
URL: http://svn.apache.org/viewvc?rev=1759894&view=rev
Log:
PIG-4974: A simple map reference fail to cast (knoguchi)
Modified:
pig/branches/branch-0.16/CHANGES.txt
pig/branches/branch-0.16/src/org/apache/pig/newplan/logical/visitor/LineageFindRelVisitor.java
pig/branches/branch-0.16/test/org/apache/pig/test/TestLineageFindRelVisitor.java
Modified: pig/branches/branch-0.16/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/CHANGES.txt?rev=1759894&r1=1759893&r2=1759894&view=diff
==============================================================================
--- pig/branches/branch-0.16/CHANGES.txt (original)
+++ pig/branches/branch-0.16/CHANGES.txt Thu Sep 8 19:41:55 2016
@@ -30,6 +30,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-4974: A simple map reference fail to cast (knoguchi)
+
PIG-4967: NPE in PigJobControl.run() when job status is null (water via daijy)
PIG-4935: TEZ_USE_CLUSTER_HADOOP_LIBS is always set to true (rohini)
Modified: pig/branches/branch-0.16/src/org/apache/pig/newplan/logical/visitor/LineageFindRelVisitor.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/src/org/apache/pig/newplan/logical/visitor/LineageFindRelVisitor.java?rev=1759894&r1=1759893&r2=1759894&view=diff
==============================================================================
--- pig/branches/branch-0.16/src/org/apache/pig/newplan/logical/visitor/LineageFindRelVisitor.java (original)
+++ pig/branches/branch-0.16/src/org/apache/pig/newplan/logical/visitor/LineageFindRelVisitor.java Thu Sep 8 19:41:55 2016
@@ -729,6 +729,15 @@ public class LineageFindRelVisitor exten
}
}
+ @Override
+ public void visit(UserFuncExpression op) throws FrontendException {
+ if(op.getFuncSpec().getClassName().equals(IdentityColumn.class.getName())) {
+ // IdentityColumn only expects one arg
+ FuncSpec funcSpec = uid2LoadFuncMap.get(op.getArguments().get(0).getFieldSchema().uid);
+ addUidLoadFuncToMap(op.getFieldSchema().uid, funcSpec);
+ }
+ }
+
/**
* if there is a null constant under casts, return it
* @param rel
Modified: pig/branches/branch-0.16/test/org/apache/pig/test/TestLineageFindRelVisitor.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/test/org/apache/pig/test/TestLineageFindRelVisitor.java?rev=1759894&r1=1759893&r2=1759894&view=diff
==============================================================================
--- pig/branches/branch-0.16/test/org/apache/pig/test/TestLineageFindRelVisitor.java (original)
+++ pig/branches/branch-0.16/test/org/apache/pig/test/TestLineageFindRelVisitor.java Thu Sep 8 19:41:55 2016
@@ -24,11 +24,16 @@ import junit.framework.Assert;
import java.io.IOException;
import java.lang.reflect.Method;
+import java.util.List;
import org.apache.pig.FuncSpec;
import org.apache.pig.LoadCaster;
+import org.apache.pig.PigServer;
import org.apache.pig.builtin.PigStorage;
import org.apache.pig.builtin.Utf8StorageConverter;
+import org.apache.pig.builtin.mock.Storage;
+import org.apache.pig.data.DataByteArray;
+import org.apache.pig.data.Tuple;
import org.apache.pig.impl.io.FileSpec;
import org.apache.pig.newplan.logical.relational.LOLoad;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
@@ -42,6 +47,12 @@ import org.junit.Test;
public class TestLineageFindRelVisitor {
+ private PigServer pig ;
+
+ public TestLineageFindRelVisitor() throws Throwable {
+ pig = new PigServer(Util.getLocalTestMode()) ;
+ }
+
public static class SillyLoadCasterWithExtraConstructor extends Utf8StorageConverter {
public SillyLoadCasterWithExtraConstructor(String ignored) {
super();
@@ -125,4 +136,32 @@ public class TestLineageFindRelVisitor {
Assert.assertEquals("Loader should be instantiated at most once.", SillyLoaderWithLoadCasterWithExtraConstructor.counter, 1);
}
+
+ @Test
+ public void testIdenticalColumnUDFForwardingLoadCaster() throws Exception {
+ Storage.Data data = Storage.resetData(pig);
+ data.set("input",
+ Storage.tuple(Storage.map(
+ "key1",new DataByteArray("aaa"),
+ "key2",new DataByteArray("bbb"),
+ "key3",new DataByteArray("ccc"))),
+ Storage.tuple(Storage.map(
+ "key1",new DataByteArray("zzz"),
+ "key2",new DataByteArray("yyy"),
+ "key3",new DataByteArray("xxx"))));
+ pig.setBatchOn();
+ pig.registerQuery("A = load 'input' using mock.Storage() as (m:[bytearray]);");
+ pig.registerQuery("B = foreach A GENERATE m#'key1' as key1, m#'key2' as key2; "
+ // this equal comparison creates implicit typecast to chararray
+ // which requires loadcaster
+ + "C = FILTER B by key1 == 'aaa' and key2 == 'bbb';");
+ pig.registerQuery("store C into 'output' using mock.Storage();");
+
+ pig.executeBatch();
+
+ List<Tuple> actualResults = data.get("output");
+ List<Tuple> expectedResults = Util.getTuplesFromConstantTupleStrings(
+ new String[] {"('aaa', 'bbb')"});
+ Util.checkQueryOutputs(actualResults.iterator(), expectedResults);
+ }
}