You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2010/05/17 19:19:45 UTC
svn commit: r945246 - in /hadoop/pig/branches/branch-0.7: CHANGES.txt
src/org/apache/pig/impl/logicalLayer/LOLoad.java
test/org/apache/pig/test/PigStorageWithSchema.java
test/org/apache/pig/test/TestLogicalPlanBuilder.java
Author: daijy
Date: Mon May 17 17:19:45 2010
New Revision: 945246
URL: http://svn.apache.org/viewvc?rev=945246&view=rev
Log:
PIG-1415: LoadFunc signature is not correct in LoadFunc.getSchema sometimes
Added:
hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/PigStorageWithSchema.java
Modified:
hadoop/pig/branches/branch-0.7/CHANGES.txt
hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/LOLoad.java
hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestLogicalPlanBuilder.java
Modified: hadoop/pig/branches/branch-0.7/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/CHANGES.txt?rev=945246&r1=945245&r2=945246&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.7/CHANGES.txt (original)
+++ hadoop/pig/branches/branch-0.7/CHANGES.txt Mon May 17 17:19:45 2010
@@ -189,6 +189,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-1415: LoadFunc signature is not correct in LoadFunc.getSchema sometimes (daijy)
+
PIG-1403: Make Pig work with remote HDFS in secure mode (daijy)
PIG-1391: pig unit tests leave behind files in temp directory because
Modified: hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/LOLoad.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/LOLoad.java?rev=945246&r1=945245&r2=945246&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/LOLoad.java (original)
+++ hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/LOLoad.java Mon May 17 17:19:45 2010
@@ -59,6 +59,7 @@ public class LOLoad extends RelationalOp
transient private Configuration conf;
private static Log log = LogFactory.getLog(LOLoad.class);
private Schema mDeterminedSchema = null;
+ private Schema scriptSchema = null;
private RequiredFieldList requiredFieldList;
private boolean mDeterminedSchemaCached = false;
@@ -149,6 +150,20 @@ public class LOLoad extends RelationalOp
if(null == mDeterminedSchema) {
mSchema = determineSchema();
}
+ if (mSchema == null) {
+ log.debug("Operator schema is null; Setting it to new schema");
+ mSchema = scriptSchema;
+ } else {
+ log.debug("Reconciling schema");
+ log.debug("mSchema: " + mSchema + " schema: " + scriptSchema);
+ try {
+ mSchema = mSchema.mergePrefixSchema(scriptSchema, true, true);
+ } catch (SchemaMergeException e) {
+ int errCode = 1019;
+ String msg = "Unable to merge schemas";
+ throw new FrontendException(msg, errCode, PigException.INPUT, false, null, e);
+ }
+ }
mIsSchemaComputed = true;
} catch (IOException ioe) {
int errCode = 1018;
@@ -182,27 +197,7 @@ public class LOLoad extends RelationalOp
*/
@Override
public void setSchema(Schema schema) throws FrontendException {
- // In general, operators don't generate their schema until they're
- // asked, so ask them to do it.
- try {
- getSchema();
- } catch (FrontendException ioe) {
- // It's fine, it just means we don't have a schema yet.
- }
- if (mSchema == null) {
- log.debug("Operator schema is null; Setting it to new schema");
- mSchema = schema;
- } else {
- log.debug("Reconciling schema");
- log.debug("mSchema: " + mSchema + " schema: " + schema);
- try {
- mSchema = mSchema.mergePrefixSchema(schema, true, true);
- } catch (SchemaMergeException e) {
- int errCode = 1019;
- String msg = "Unable to merge schemas";
- throw new FrontendException(msg, errCode, PigException.INPUT, false, null, e);
- }
- }
+ scriptSchema = schema;
}
Added: hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/PigStorageWithSchema.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/PigStorageWithSchema.java?rev=945246&view=auto
==============================================================================
--- hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/PigStorageWithSchema.java (added)
+++ hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/PigStorageWithSchema.java Mon May 17 17:19:45 2010
@@ -0,0 +1,46 @@
+package org.apache.pig.test;
+
+import java.io.IOException;
+
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.pig.Expression;
+import org.apache.pig.LoadMetadata;
+import org.apache.pig.ResourceSchema;
+import org.apache.pig.ResourceStatistics;
+import org.apache.pig.builtin.PigStorage;
+
+public class PigStorageWithSchema extends PigStorage implements LoadMetadata {
+ private String signature;
+
+ @Override
+ public String[] getPartitionKeys(String location, Job job)
+ throws IOException {
+ return null;
+ }
+
+ @Override
+ public ResourceSchema getSchema(String location, Job job)
+ throws IOException {
+ return null;
+ }
+
+ @Override
+ public ResourceStatistics getStatistics(String location, Job job)
+ throws IOException {
+ return null;
+ }
+
+ @Override
+ public void setPartitionFilter(Expression partitionFilter)
+ throws IOException {
+ }
+
+ @Override
+ public void setUDFContextSignature(String signature) {
+ this.signature = signature;
+ }
+
+ public String getUDFContextSignature() {
+ return signature;
+ }
+}
Modified: hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestLogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=945246&r1=945245&r2=945246&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestLogicalPlanBuilder.java (original)
+++ hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestLogicalPlanBuilder.java Mon May 17 17:19:45 2010
@@ -2144,6 +2144,14 @@ public class TestLogicalPlanBuilder exte
"not occur", true, exceptionThrown);
}
+ @Test
+ public void testLoaderSignature() {
+ LogicalPlan plan = buildPlan(" a = load '1.txt' using org.apache.pig.test.PigStorageWithSchema() as (a0:int, a1:int);");
+ assertTrue(((PigStorageWithSchema)((LOLoad)plan.getLeaves().get(0)).getLoadFunc()).getUDFContextSignature().equals("a"));
+ plan = buildPlan(" b = load '1.txt' using org.apache.pig.test.PigStorageWithSchema();");
+ assertTrue(((PigStorageWithSchema)((LOLoad)plan.getLeaves().get(0)).getLoadFunc()).getUDFContextSignature().equals("b"));
+ }
+
private void printPlan(LogicalPlan lp) {
LOPrinter graphPrinter = new LOPrinter(System.err, lp);
System.err.println("Printing the logical plan");