You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2013/05/02 02:45:49 UTC
svn commit: r1478243 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/PigServer.java
src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
test/org/apache/pig/test/TestEvalPipelineLocal.java
Author: daijy
Date: Thu May 2 00:45:48 2013
New Revision: 1478243
URL: http://svn.apache.org/r1478243
Log:
PIG-2970: Nested foreach getting incorrect schema when having unrelated inner query
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/PigServer.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1478243&r1=1478242&r2=1478243&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu May 2 00:45:48 2013
@@ -170,6 +170,8 @@ PIG-3013: BinInterSedes improve chararra
BUG FIXES
+PIG-2970: Nested foreach getting incorrect schema when having unrelated inner query (daijy)
+
PIG-3304: XMLLoader in piggybank does not work with inline closed tags (aseldawy via daijy)
PIG-3028: testGrunt dev test needs some command filters to run correctly without cygwin (jgordon via gates)
Modified: pig/trunk/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigServer.java?rev=1478243&r1=1478242&r2=1478243&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigServer.java (original)
+++ pig/trunk/src/org/apache/pig/PigServer.java Thu May 2 00:45:48 2013
@@ -84,6 +84,8 @@ import org.apache.pig.newplan.logical.ex
import org.apache.pig.newplan.logical.expression.LogicalExpressionVisitor;
import org.apache.pig.newplan.logical.expression.ScalarExpression;
import org.apache.pig.newplan.logical.optimizer.AllExpressionVisitor;
+import org.apache.pig.newplan.logical.optimizer.DanglingNestedNodeRemover;
+import org.apache.pig.newplan.logical.optimizer.UidResetter;
import org.apache.pig.newplan.logical.relational.LOForEach;
import org.apache.pig.newplan.logical.relational.LOLoad;
import org.apache.pig.newplan.logical.relational.LOStore;
@@ -1379,14 +1381,13 @@ public class PigServer {
}
private LogicalRelationalOperator getOperatorForAlias(String alias) throws IOException {
- currDAG.parseQuery();
+ buildStorePlan (alias);
LogicalRelationalOperator op = (LogicalRelationalOperator)currDAG.getOperator( alias );
if( op == null ) {
int errCode = 1005;
String msg = "No plan for " + alias + " to describe";
throw new FrontendException(msg, errCode, PigException.INPUT, false, null);
}
- currDAG.compile();
return op;
}
@@ -1689,6 +1690,9 @@ public class PigServer {
}
private void compile(LogicalPlan lp) throws FrontendException {
+ DanglingNestedNodeRemover DanglingNestedNodeRemover = new DanglingNestedNodeRemover( lp );
+ DanglingNestedNodeRemover.visit();
+
new ColumnAliasConversionVisitor(lp).visit();
new SchemaAliasVisitor(lp).visit();
new ScalarVisitor(lp, pigContext, scope).visit();
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java?rev=1478243&r1=1478242&r2=1478243&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java Thu May 2 00:45:48 2013
@@ -240,9 +240,6 @@ public class HExecutionEngine {
pod.visit();
}
- DanglingNestedNodeRemover DanglingNestedNodeRemover = new DanglingNestedNodeRemover( plan );
- DanglingNestedNodeRemover.visit();
-
UidResetter uidResetter = new UidResetter( plan );
uidResetter.visit();
Modified: pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java?rev=1478243&r1=1478242&r2=1478243&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestEvalPipelineLocal.java Thu May 2 00:45:48 2013
@@ -57,6 +57,7 @@ import org.apache.pig.impl.io.PigFile;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.util.Pair;
import org.apache.pig.impl.util.UDFContext;
+import org.apache.pig.impl.util.Utils;
import org.apache.pig.test.utils.Identity;
import org.junit.Before;
import org.junit.Test;
@@ -1123,4 +1124,23 @@ public class TestEvalPipelineLocal {
Assert.assertTrue(iter.next().toString().equals("(0)"));
Assert.assertFalse(iter.hasNext());
}
+
+ @Test
+ // See PIG-2970
+ public void testDescribeDanglingBranch() throws Throwable {
+ File f1 = createFile(new String[]{"NYSE\tIBM", "NASDAQ\tYHOO", "NASDAQ\tMSFT"});
+ pigServer.registerQuery("daily = load '" + Util.generateURI(f1.toString(), pigServer.getPigContext())
+ +"' as (exchange, symbol);");
+ pigServer.registerQuery("grpd = group daily by exchange;");
+ pigServer.registerQuery("unique = foreach grpd { sym = daily.symbol; uniq_sym = distinct sym; uniq_sym = distinct sym; generate group, daily;};");
+ pigServer.registerQuery("zzz = foreach unique generate group;");
+ Schema dumpedSchema = pigServer.dumpSchema("zzz") ;
+ Schema expectedSchema = Utils.getSchemaFromString(
+ "group: bytearray");
+ Assert.assertEquals(expectedSchema, dumpedSchema);
+ Iterator<Tuple> iter = pigServer.openIterator("zzz");
+ Assert.assertTrue(iter.next().toString().equals("(NYSE)"));
+ Assert.assertTrue(iter.next().toString().equals("(NASDAQ)"));
+ Assert.assertFalse(iter.hasNext());
+ }
}