You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2013/03/18 18:26:43 UTC
svn commit: r1457882 - in /pig/branches/branch-0.11: ./
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/
src/org/apache/pig/impl/io/ src/org/apache/pig/pen/ test/org/apache/pig/test/
Author: daijy
Date: Mon Mar 18 17:26:43 2013
New Revision: 1457882
URL: http://svn.apache.org/r1457882
Log:
PIG-3132: NPE when illustrating a relation with HCatLoader
Modified:
pig/branches/branch-0.11/CHANGES.txt
pig/branches/branch-0.11/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLoad.java
pig/branches/branch-0.11/src/org/apache/pig/impl/io/ReadToEndLoader.java
pig/branches/branch-0.11/src/org/apache/pig/pen/LocalMapReduceSimulator.java
pig/branches/branch-0.11/test/org/apache/pig/test/TestExampleGenerator.java
Modified: pig/branches/branch-0.11/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.11/CHANGES.txt?rev=1457882&r1=1457881&r2=1457882&view=diff
==============================================================================
--- pig/branches/branch-0.11/CHANGES.txt (original)
+++ pig/branches/branch-0.11/CHANGES.txt Mon Mar 18 17:26:43 2013
@@ -36,6 +36,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-3132: NPE when illustrating a relation with HCatLoader (daijy)
+
PIG-3194: Changes to ObjectSerializer.java break compatibility with Hadoop 0.20.2 (prkommireddi via dvryaboy)
PIG-3241: ConcurrentModificationException in POPartialAgg (dvryaboy)
Modified: pig/branches/branch-0.11/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLoad.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.11/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLoad.java?rev=1457882&r1=1457881&r2=1457882&view=diff
==============================================================================
--- pig/branches/branch-0.11/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLoad.java (original)
+++ pig/branches/branch-0.11/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLoad.java Mon Mar 18 17:26:43 2013
@@ -93,7 +93,7 @@ public class POLoad extends PhysicalOper
loader = new ReadToEndLoader((LoadFunc)
PigContext.instantiateFuncFromSpec(lFile.getFuncSpec()),
ConfigurationUtil.toConfiguration(pc.getProperties()),
- lFile.getFileName(),0);
+ lFile.getFileName(),0, signature);
}
/**
Modified: pig/branches/branch-0.11/src/org/apache/pig/impl/io/ReadToEndLoader.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.11/src/org/apache/pig/impl/io/ReadToEndLoader.java?rev=1457882&r1=1457881&r2=1457882&view=diff
==============================================================================
--- pig/branches/branch-0.11/src/org/apache/pig/impl/io/ReadToEndLoader.java (original)
+++ pig/branches/branch-0.11/src/org/apache/pig/impl/io/ReadToEndLoader.java Mon Mar 18 17:26:43 2013
@@ -106,6 +106,8 @@ public class ReadToEndLoader extends Loa
private InputFormat inputFormat = null;
private PigContext pigContext;
+
+ private String udfContextSignature = null;
/**
* @param wrappedLoadFunc
@@ -133,6 +135,16 @@ public class ReadToEndLoader extends Loa
this.pigContext = pigContext;
init();
}
+
+ public ReadToEndLoader(LoadFunc wrappedLoadFunc, Configuration conf,
+ String inputLocation, int splitIndex, String signature) throws IOException {
+ this.udfContextSignature = signature;
+ this.wrappedLoadFunc = wrappedLoadFunc;
+ this.inputLocation = inputLocation;
+ this.conf = conf;
+ this.curSplitIndex = splitIndex;
+ init();
+ }
/**
* This constructor takes an array of split indexes (toReadSplitIdxs) of the
@@ -167,6 +179,7 @@ public class ReadToEndLoader extends Loa
// let's initialize the wrappedLoadFunc
Job job = new Job(conf);
+ wrappedLoadFunc.setUDFContextSignature(this.udfContextSignature);
wrappedLoadFunc.setLocation(inputLocation,
job);
// The above setLocation call could write to the conf within
@@ -277,7 +290,7 @@ public class ReadToEndLoader extends Loa
@Override
public void setLocation(String location, Job job) throws IOException {
- //no-op
+ wrappedLoadFunc.setLocation(location, job);
}
@Override
@@ -313,4 +326,9 @@ public class ReadToEndLoader extends Loa
((LoadMetadata) wrappedLoadFunc).setPartitionFilter(partitionFilter);
}
}
+
+ @Override
+ public void setUDFContextSignature(String signature) {
+ this.udfContextSignature = signature;
+ }
}
Modified: pig/branches/branch-0.11/src/org/apache/pig/pen/LocalMapReduceSimulator.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.11/src/org/apache/pig/pen/LocalMapReduceSimulator.java?rev=1457882&r1=1457881&r2=1457882&view=diff
==============================================================================
--- pig/branches/branch-0.11/src/org/apache/pig/pen/LocalMapReduceSimulator.java (original)
+++ pig/branches/branch-0.11/src/org/apache/pig/pen/LocalMapReduceSimulator.java Mon Mar 18 17:26:43 2013
@@ -105,6 +105,7 @@ public class LocalMapReduceSimulator {
// jc is null only when mrp.size == 0
boolean needFileInput;
final ArrayList<OperatorKey> emptyInpTargets = new ArrayList<OperatorKey>();
+ pc.getProperties().setProperty("pig.illustrating", "true");
while(mrp.size() != 0) {
jc = jcc.compile(mrp, "Illustrator");
if(jc == null) {
Modified: pig/branches/branch-0.11/test/org/apache/pig/test/TestExampleGenerator.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.11/test/org/apache/pig/test/TestExampleGenerator.java?rev=1457882&r1=1457881&r2=1457882&view=diff
==============================================================================
--- pig/branches/branch-0.11/test/org/apache/pig/test/TestExampleGenerator.java (original)
+++ pig/branches/branch-0.11/test/org/apache/pig/test/TestExampleGenerator.java Mon Mar 18 17:26:43 2013
@@ -33,6 +33,7 @@ import org.apache.pig.backend.executione
import org.apache.pig.data.DataBag;
import org.apache.pig.impl.PigContext;
import org.apache.pig.newplan.Operator;
+import org.apache.pig.test.utils.UDFContextTestLoaderWithSignature;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -394,5 +395,14 @@ public class TestExampleGenerator {
assertNotNull(derivedData);
}
+
+ @Test
+ public void testLoaderWithContext() throws Exception {
+ PigServer pigServer = new PigServer(pigContext);
+ pigServer.registerQuery("A = load " + A.toString() + " using " + UDFContextTestLoaderWithSignature.class.getName() + "('a') as (x, y);");
+ Map<Operator, DataBag> derivedData = pigServer.getExamples("A");
+
+ assertNotNull(derivedData);
+ }
}