You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2010/12/10 20:59:31 UTC
svn commit: r1044495 - in /pig/branches/branch-0.8: CHANGES.txt
src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java
test/org/apache/pig/test/TestEvalPipeline2.java
Author: daijy
Date: Fri Dec 10 19:59:31 2010
New Revision: 1044495
URL: http://svn.apache.org/viewvc?rev=1044495&view=rev
Log:
PIG-1761: New logical plan: Exception when bag dereference in the middle of expression
Modified:
pig/branches/branch-0.8/CHANGES.txt
pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java
pig/branches/branch-0.8/test/org/apache/pig/test/TestEvalPipeline2.java
Modified: pig/branches/branch-0.8/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.8/CHANGES.txt?rev=1044495&r1=1044494&r2=1044495&view=diff
==============================================================================
--- pig/branches/branch-0.8/CHANGES.txt (original)
+++ pig/branches/branch-0.8/CHANGES.txt Fri Dec 10 19:59:31 2010
@@ -22,6 +22,8 @@ Release 0.8.0
INCOMPATIBLE CHANGES
+PIG-1518: multi file input format for loaders (yanz via rding)
+
PIG-1249: Safe-guards against misconfigured Pig scripts without PARALLEL keyword (zjffdu vi olgan)
IMPROVEMENTS
@@ -77,8 +79,6 @@ PIG-1483: [piggybank] Add HadoopJobHisto
PIG-1555: [piggybank] add CSV Loader (dvryaboy)
-PIG-1518: multi file input format for loaders (yanz via rding)
-
PIG-1501: need to investigate the impact of compression on pig performance (yanz via thejas)
PIG-1497: Mandatory rule PartitionFilterOptimizer (xuefuz via daijy)
@@ -211,6 +211,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-1761: New logical plan: Exception when bag dereference in the middle of expression (daijy)
+
PIG-1760: Need to report progress in all databags (rding)
PIG-1709: Skewed join use fewer reducer for extreme large key (daijy)
Modified: pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java?rev=1044495&r1=1044494&r2=1044495&view=diff
==============================================================================
--- pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java (original)
+++ pig/branches/branch-0.8/src/org/apache/pig/newplan/logical/ForeachInnerPlanVisitor.java Fri Dec 10 19:59:31 2010
@@ -136,11 +136,12 @@ public class ForeachInnerPlanVisitor ext
}
}
- // This case occurs when there are two projects one after another
+ // If project is after an expression, translate it into a dereference.
+ // One particular case is when there are two projects one after another
// These projects in combination project a column (bag) out of a tuple
// and then project a column out of this projected bag
// Here we merge these two projects into one BagDereferenceExpression
- else if( op instanceof LOProject ) {
+ else if( op instanceof ExpressionOperator ) {
LogicalExpression expOper = exprOpsMap.get(op);
if (expOper!=null) {
@@ -152,6 +153,7 @@ public class ForeachInnerPlanVisitor ext
exprPlan.connect(dereferenceExp, expOper);
}
} else {
+ // If project is after a relational operator
if (op instanceof RelationalOperator && project.isSendEmptyBagOnEOP()) {
LogicalOperator currentOp = op;
while (currentOp instanceof RelationalOperator) {
Modified: pig/branches/branch-0.8/test/org/apache/pig/test/TestEvalPipeline2.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.8/test/org/apache/pig/test/TestEvalPipeline2.java?rev=1044495&r1=1044494&r2=1044495&view=diff
==============================================================================
--- pig/branches/branch-0.8/test/org/apache/pig/test/TestEvalPipeline2.java (original)
+++ pig/branches/branch-0.8/test/org/apache/pig/test/TestEvalPipeline2.java Fri Dec 10 19:59:31 2010
@@ -948,4 +948,21 @@ public class TestEvalPipeline2 extends T
assertFalse(iter.hasNext());
}
+
+ // See PIG-1761
+ @Test
+ public void testBagDereferenceInMiddle() throws Exception{
+ String[] input1 = {
+ "foo@apache#44",
+ };
+
+ Util.createInputFile(cluster, "table_testBagDereferenceInMiddle", input1);
+ pigServer.registerQuery("a = load 'table_testBagDereferenceInMiddle' as (a0:chararray);");
+ pigServer.registerQuery("b = foreach a generate UPPER(REGEX_EXTRACT_ALL(a0, '.*@(.*)#.*').$0);");
+
+ Iterator<Tuple> iter = pigServer.openIterator("b");
+ Tuple t = iter.next();
+ assertTrue(t.size()==1);
+ assertTrue(t.get(0).equals("APACHE"));
+ }
}