You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by zj...@apache.org on 2010/06/21 08:46:43 UTC
svn commit: r956440 - in /hadoop/pig/trunk: ./
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/
test/org/apache/pig/test/
Author: zjffdu
Date: Mon Jun 21 02:10:35 2010
New Revision: 956440
URL: http://svn.apache.org/viewvc?rev=956440&view=rev
Log:
PIG-1221: Filter equality does not work for tuples
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/EqualToExpr.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/NotEqualToExpr.java
hadoop/pig/trunk/test/org/apache/pig/test/TestEqualTo.java
hadoop/pig/trunk/test/org/apache/pig/test/TestNotEqualTo.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=956440&r1=956439&r2=956440&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Mon Jun 21 02:10:35 2010
@@ -86,6 +86,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-1221: Filter equality does not work for tuples (zjffdu)
+
PIG-1456: TestMultiQuery takes a long time to run (rding)
PIG-1457: Pig will run complete zebra test even we give -Dtestcase=xxx (daijy)
Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/EqualToExpr.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/EqualToExpr.java?rev=956440&r1=956439&r2=956440&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/EqualToExpr.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/EqualToExpr.java Mon Jun 21 02:10:35 2010
@@ -17,6 +17,8 @@
*/
package org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators;
+import java.util.HashMap;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -124,7 +126,26 @@ public class EqualToExpr extends BinaryC
return doComparison(left, right);
}
-
+ case DataType.TUPLE: {
+ Result r = accumChild(null, dummyTuple);
+ if (r != null) {
+ return r;
+ }
+ left = lhs.getNext(dummyTuple);
+ right = rhs.getNext(dummyTuple);
+ return doComparison(left, right);
+ }
+
+ case DataType.MAP: {
+ Result r = accumChild(null, dummyMap);
+ if (r != null) {
+ return r;
+ }
+ left = lhs.getNext(dummyMap);
+ right = rhs.getNext(dummyMap);
+ return doComparison(left, right);
+ }
+
default: {
int errCode = 2067;
String msg = this.getClass().getSimpleName() + " does not know how to " +
@@ -136,7 +157,7 @@ public class EqualToExpr extends BinaryC
}
@SuppressWarnings("unchecked")
- private Result doComparison(Result left, Result right) {
+ private Result doComparison(Result left, Result right) throws ExecException {
if (trueRef == null) initializeRefs();
if (left.returnStatus != POStatus.STATUS_OK) return left;
if (right.returnStatus != POStatus.STATUS_OK) return right;
@@ -147,13 +168,24 @@ public class EqualToExpr extends BinaryC
left.returnStatus = POStatus.STATUS_NULL;
return left;
}
- assert(left instanceof Comparable);
- assert(right instanceof Comparable);
- if (((Comparable)left.result).compareTo((Comparable)right.result) == 0) {
- left.result = trueRef;
- } else {
- left.result = falseRef;
+
+ if (left.result instanceof Comparable && right.result instanceof Comparable){
+ if (((Comparable)left.result).compareTo((Comparable)right.result) == 0) {
+ left.result = trueRef;
+ } else {
+ left.result = falseRef;
+ }
+ }else if (left.result instanceof HashMap && right.result instanceof HashMap){
+ HashMap leftMap=(HashMap)left.result;
+ HashMap rightMap=(HashMap)right.result;
+ if (leftMap.equals(rightMap))
+ left.result = trueRef;
+ else
+ left.result = falseRef;
+ }else{
+ throw new ExecException("The left side and right side has the different types");
}
+
return left;
}
Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/NotEqualToExpr.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/NotEqualToExpr.java?rev=956440&r1=956439&r2=956440&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/NotEqualToExpr.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/NotEqualToExpr.java Mon Jun 21 02:10:35 2010
@@ -17,6 +17,8 @@
*/
package org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators;
+import java.util.HashMap;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -123,7 +125,25 @@ public class NotEqualToExpr extends Bina
right = rhs.getNext(dummyString);
return doComparison(left, right);
}
-
+ case DataType.TUPLE: {
+ Result r = accumChild(null, dummyTuple);
+ if (r != null) {
+ return r;
+ }
+ left = lhs.getNext(dummyTuple);
+ right = rhs.getNext(dummyTuple);
+ return doComparison(left, right);
+ }
+
+ case DataType.MAP: {
+ Result r = accumChild(null, dummyMap);
+ if (r != null) {
+ return r;
+ }
+ left = lhs.getNext(dummyMap);
+ right = rhs.getNext(dummyMap);
+ return doComparison(left, right);
+ }
default: {
int errCode = 2067;
@@ -136,7 +156,7 @@ public class NotEqualToExpr extends Bina
}
@SuppressWarnings("unchecked")
- private Result doComparison(Result left, Result right) {
+ private Result doComparison(Result left, Result right) throws ExecException {
if (trueRef == null) initializeRefs();
if (left.returnStatus != POStatus.STATUS_OK) return left;
if (right.returnStatus != POStatus.STATUS_OK) return right;
@@ -147,12 +167,22 @@ public class NotEqualToExpr extends Bina
left.returnStatus = POStatus.STATUS_NULL;
return left;
}
- assert(left instanceof Comparable);
- assert(right instanceof Comparable);
- if (((Comparable)left.result).compareTo((Comparable)right.result) != 0) {
- left.result = trueRef;
- } else {
- left.result = falseRef;
+
+ if (left.result instanceof Comparable && right.result instanceof Comparable){
+ if (((Comparable)left.result).compareTo((Comparable)right.result) != 0) {
+ left.result = trueRef;
+ } else {
+ left.result = falseRef;
+ }
+ }else if (left.result instanceof HashMap && right.result instanceof HashMap){
+ HashMap leftMap=(HashMap)left.result;
+ HashMap rightMap=(HashMap)right.result;
+ if (leftMap.equals(rightMap))
+ left.result = falseRef;
+ else
+ left.result = trueRef;
+ }else{
+ throw new ExecException("The left side and right side has the different types");
}
return left;
}
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestEqualTo.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestEqualTo.java?rev=956440&r1=956439&r2=956440&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestEqualTo.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestEqualTo.java Mon Jun 21 02:10:35 2010
@@ -17,6 +17,7 @@
*/
package org.apache.pig.test;
+import java.util.HashMap;
import java.util.Map;
import java.util.Random;
@@ -25,6 +26,7 @@ import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.ConstantExpression;
@@ -270,6 +272,81 @@ public class TestEqualTo extends junit.f
assertTrue((Boolean)r.result);
}
+ @Test
+ public void testTupleEq() throws ExecException{
+ Tuple tuple_1=TupleFactory.getInstance().newTuple("item_1");
+ Tuple tuple_2=TupleFactory.getInstance().newTuple("item_1");
+ ConstantExpression lt = GenPhyOp.exprConst();
+ lt.setValue(tuple_1);
+ ConstantExpression rt = GenPhyOp.exprConst();
+ rt.setValue(tuple_2);
+ EqualToExpr g = GenPhyOp.compEqualToExpr();
+ g.setLhs(lt);
+ g.setRhs(rt);
+ g.setOperandType(DataType.TUPLE);
+ Result r = g.getNext(new Boolean(true));
+ assertEquals(POStatus.STATUS_OK, r.returnStatus);
+ assertTrue((Boolean)r.result);
+ }
+
+ @Test
+ public void testTupleNe() throws ExecException{
+ Tuple tuple_1=TupleFactory.getInstance().newTuple("item_1");
+ Tuple tuple_2=TupleFactory.getInstance().newTuple("item_2");
+ ConstantExpression lt = GenPhyOp.exprConst();
+ lt.setValue(tuple_1);
+ ConstantExpression rt = GenPhyOp.exprConst();
+ rt.setValue(tuple_2);
+ EqualToExpr g = GenPhyOp.compEqualToExpr();
+ g.setLhs(lt);
+ g.setRhs(rt);
+ g.setOperandType(DataType.TUPLE);
+ Result r = g.getNext(new Boolean(true));
+ assertEquals(POStatus.STATUS_OK, r.returnStatus);
+ assertFalse((Boolean)r.result);
+ }
+
+ @Test
+ public void testMapEq() throws ExecException{
+ Map map_1=new HashMap();
+ map_1.put("key_1", "value_1");
+ Map map_2=new HashMap();
+ map_2.put("key_1", "value_1");
+
+ ConstantExpression lt = GenPhyOp.exprConst();
+ lt.setValue(map_1);
+ ConstantExpression rt = GenPhyOp.exprConst();
+ rt.setValue(map_2);
+ EqualToExpr g = GenPhyOp.compEqualToExpr();
+ g.setLhs(lt);
+ g.setRhs(rt);
+ g.setOperandType(DataType.MAP);
+ Result r = g.getNext(new Boolean(true));
+ assertEquals(POStatus.STATUS_OK, r.returnStatus);
+ assertTrue((Boolean)r.result);
+ }
+
+ @Test
+ public void testMapNe() throws ExecException{
+ Map map_1=new HashMap();
+ map_1.put("key_1", "value_1");
+ Map map_2=new HashMap();
+ map_2.put("key_1", "value_2");
+
+ Tuple tuple_2=TupleFactory.getInstance().newTuple("item_2");
+ ConstantExpression lt = GenPhyOp.exprConst();
+ lt.setValue(map_1);
+ ConstantExpression rt = GenPhyOp.exprConst();
+ rt.setValue(map_2);
+ EqualToExpr g = GenPhyOp.compEqualToExpr();
+ g.setLhs(lt);
+ g.setRhs(rt);
+ g.setOperandType(DataType.MAP);
+ Result r = g.getNext(new Boolean(true));
+ assertEquals(POStatus.STATUS_OK, r.returnStatus);
+ assertFalse((Boolean)r.result);
+ }
+
@Test
public void testDataByteArrayAndNullValues() throws Exception {
checkNullValues( DataType.BYTEARRAY, new DataByteArray("b") );
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestNotEqualTo.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestNotEqualTo.java?rev=956440&r1=956439&r2=956440&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestNotEqualTo.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestNotEqualTo.java Mon Jun 21 02:10:35 2010
@@ -17,6 +17,7 @@
*/
package org.apache.pig.test;
+import java.util.HashMap;
import java.util.Map;
import java.util.Random;
@@ -25,9 +26,11 @@ import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.ConstantExpression;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.EqualToExpr;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.NotEqualToExpr;
import org.apache.pig.test.utils.GenPhyOp;
import org.apache.pig.test.utils.GenRandomData;
@@ -260,7 +263,81 @@ public class TestNotEqualTo extends juni
assertFalse((Boolean)r.result);
}
-
+ @Test
+ public void testTupleEq() throws ExecException{
+ Tuple tuple_1=TupleFactory.getInstance().newTuple("item_1");
+ Tuple tuple_2=TupleFactory.getInstance().newTuple("item_1");
+ ConstantExpression lt = GenPhyOp.exprConst();
+ lt.setValue(tuple_1);
+ ConstantExpression rt = GenPhyOp.exprConst();
+ rt.setValue(tuple_2);
+ NotEqualToExpr g = GenPhyOp.compNotEqualToExpr();
+ g.setLhs(lt);
+ g.setRhs(rt);
+ g.setOperandType(DataType.TUPLE);
+ Result r = g.getNext(new Boolean(true));
+ assertEquals(POStatus.STATUS_OK, r.returnStatus);
+ assertFalse((Boolean)r.result);
+ }
+
+ @Test
+ public void testTupleNe() throws ExecException{
+ Tuple tuple_1=TupleFactory.getInstance().newTuple("item_1");
+ Tuple tuple_2=TupleFactory.getInstance().newTuple("item_2");
+ ConstantExpression lt = GenPhyOp.exprConst();
+ lt.setValue(tuple_1);
+ ConstantExpression rt = GenPhyOp.exprConst();
+ rt.setValue(tuple_2);
+ NotEqualToExpr g = GenPhyOp.compNotEqualToExpr();
+ g.setLhs(lt);
+ g.setRhs(rt);
+ g.setOperandType(DataType.TUPLE);
+ Result r = g.getNext(new Boolean(true));
+ assertEquals(POStatus.STATUS_OK, r.returnStatus);
+ assertTrue((Boolean)r.result);
+ }
+
+ @Test
+ public void testMapEq() throws ExecException{
+ Map map_1=new HashMap();
+ map_1.put("key_1", "value_1");
+ Map map_2=new HashMap();
+ map_2.put("key_1", "value_1");
+
+ ConstantExpression lt = GenPhyOp.exprConst();
+ lt.setValue(map_1);
+ ConstantExpression rt = GenPhyOp.exprConst();
+ rt.setValue(map_2);
+ NotEqualToExpr g = GenPhyOp.compNotEqualToExpr();
+ g.setLhs(lt);
+ g.setRhs(rt);
+ g.setOperandType(DataType.MAP);
+ Result r = g.getNext(new Boolean(true));
+ assertEquals(POStatus.STATUS_OK, r.returnStatus);
+ assertFalse((Boolean)r.result);
+ }
+
+ @Test
+ public void testMapNe() throws ExecException{
+ Map map_1=new HashMap();
+ map_1.put("key_1", "value_1");
+ Map map_2=new HashMap();
+ map_2.put("key_1", "value_2");
+
+ Tuple tuple_2=TupleFactory.getInstance().newTuple("item_2");
+ ConstantExpression lt = GenPhyOp.exprConst();
+ lt.setValue(map_1);
+ ConstantExpression rt = GenPhyOp.exprConst();
+ rt.setValue(map_2);
+ NotEqualToExpr g = GenPhyOp.compNotEqualToExpr();
+ g.setLhs(lt);
+ g.setRhs(rt);
+ g.setOperandType(DataType.MAP);
+ Result r = g.getNext(new Boolean(true));
+ assertEquals(POStatus.STATUS_OK, r.returnStatus);
+ assertTrue((Boolean)r.result);
+ }
+
@Test
public void testDataByteArrayAndNullValues() throws Exception {