You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2010/12/10 21:00:57 UTC
svn commit: r1044496 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java
test/org/apache/pig/test/TestEvalPipeline2.java
Author: daijy
Date: Fri Dec 10 20:00:57 2010
New Revision: 1044496
URL: http://svn.apache.org/viewvc?rev=1044496&view=rev
Log:
PIG-1761: New logical plan: Exception when bag dereference in the middle of expression
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java
pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1044496&r1=1044495&r2=1044496&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Dec 10 20:00:57 2010
@@ -59,6 +59,8 @@ Release 0.8.0 - Unreleased
INCOMPATIBLE CHANGES
+PIG-1518: multi file input format for loaders (yanz via rding)
+
PIG-1249: Safe-guards against misconfigured Pig scripts without PARALLEL keyword (zjffdu vi olgan)
IMPROVEMENTS
@@ -100,8 +102,6 @@ PIG-1483: [piggybank] Add HadoopJobHisto
PIG-1555: [piggybank] add CSV Loader (dvryaboy)
-PIG-1518: multi file input format for loaders (yanz via rding)
-
PIG-1501: need to investigate the impact of compression on pig performance (yanz via thejas)
PIG-1497: Mandatory rule PartitionFilterOptimizer (xuefuz via daijy)
@@ -234,6 +234,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-1761: New logical plan: Exception when bag dereference in the middle of expression (daijy)
+
PIG-1757: After split combination, the number of maps may vary slightly (yanz)
PIG-1760: Need to report progress in all databags (rding)
Modified: pig/trunk/src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java?rev=1044496&r1=1044495&r2=1044496&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java Fri Dec 10 20:00:57 2010
@@ -136,11 +136,12 @@ public class ForeachInnerPlanVisitor ext
}
}
- // This case occurs when there are two projects one after another
+ // If project is after an expression, translate it into a dereference.
+ // One particular case is when there are two projects one after another
// These projects in combination project a column (bag) out of a tuple
// and then project a column out of this projected bag
// Here we merge these two projects into one BagDereferenceExpression
- else if( op instanceof LOProject ) {
+ else if( op instanceof ExpressionOperator ) {
LogicalExpression expOper = exprOpsMap.get(op);
if (expOper!=null) {
@@ -152,6 +153,7 @@ public class ForeachInnerPlanVisitor ext
exprPlan.connect(dereferenceExp, expOper);
}
} else {
+ // If project is after a relational operator
if (op instanceof RelationalOperator && project.isSendEmptyBagOnEOP()) {
LogicalOperator currentOp = op;
while (currentOp instanceof RelationalOperator) {
Modified: pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java?rev=1044496&r1=1044495&r2=1044496&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java Fri Dec 10 20:00:57 2010
@@ -966,4 +966,21 @@ public class TestEvalPipeline2 extends T
fail();
}
+
+ // See PIG-1761
+ @Test
+ public void testBagDereferenceInMiddle() throws Exception{
+ String[] input1 = {
+ "foo@apache#44",
+ };
+
+ Util.createInputFile(cluster, "table_testBagDereferenceInMiddle", input1);
+ pigServer.registerQuery("a = load 'table_testBagDereferenceInMiddle' as (a0:chararray);");
+ pigServer.registerQuery("b = foreach a generate UPPER(REGEX_EXTRACT_ALL(a0, '.*@(.*)#.*').$0);");
+
+ Iterator<Tuple> iter = pigServer.openIterator("b");
+ Tuple t = iter.next();
+ assertTrue(t.size()==1);
+ assertTrue(t.get(0).equals("APACHE"));
+ }
}