You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by sm...@apache.org on 2009/07/31 23:13:05 UTC
svn commit: r799749 - in /hadoop/pig/trunk: ./
src/org/apache/pig/data/parser/ test/org/apache/pig/test/
test/org/apache/pig/test/utils/
Author: sms
Date: Fri Jul 31 21:13:03 2009
New Revision: 799749
URL: http://svn.apache.org/viewvc?rev=799749&view=rev
Log:
PIG-880: Order by is borken with complex fields
Added:
hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/data/parser/TextDataParser.jjt
hadoop/pig/trunk/test/org/apache/pig/test/TestConversions.java
hadoop/pig/trunk/test/org/apache/pig/test/TestStore.java
hadoop/pig/trunk/test/org/apache/pig/test/utils/TestHelper.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=799749&r1=799748&r2=799749&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Fri Jul 31 21:13:03 2009
@@ -44,6 +44,8 @@
BUG FIXES
+ PIG-880: Order by is borken with complex fields (sms)
+
PIG-773: Empty complex constants (empty bag, empty tuple and empty map)
should be supported (ashutoshc via sms)
Modified: hadoop/pig/trunk/src/org/apache/pig/data/parser/TextDataParser.jjt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/data/parser/TextDataParser.jjt?rev=799749&r1=799748&r2=799749&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/data/parser/TextDataParser.jjt (original)
+++ hadoop/pig/trunk/src/org/apache/pig/data/parser/TextDataParser.jjt Fri Jul 31 21:13:03 2009
@@ -44,6 +44,7 @@
import org.apache.pig.data.Tuple;
import org.apache.pig.data.BagFactory;
import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataByteArray;
public class TextDataParser {
@@ -141,13 +142,13 @@
void KeyValuePair(Map<String, Object> keyValues) :
{
String key = null;
- Object value = null;
+ String value = null;
log.trace("Entering KeyValuePair");
}
{
- (key = StringDatum() "#" value = Datum())
+ (key = StringDatum() "#" value = StringDatum())
{
- keyValues.put(key, value);
+ keyValues.put(key, new DataByteArray(value.getBytes("UTF-8")));
log.trace("Exiting KeyValuePair");
}
@@ -161,11 +162,20 @@
}
{
(
- t = <STRING> { s = t.image; }
-| t = <NULL> {}
+ (
+ t = <SIGNEDINTEGER>
+| t = <LONGINTEGER>
+| t = <FLOATNUMBER>
+| t = <DOUBLENUMBER>
+| t = <STRING>
+ )
+ {
+ s = t.image;
+ }
+
)
{
- log.trace("Exiting String with " + t.image);
+ log.trace("Exiting String with " + s);
return s;
}
}
@@ -195,7 +205,6 @@
{
obj = new Double(Double.parseDouble(t.image));
}
-| t = <NULL> {}
| t = <STRING>
{
obj = t.image;
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestConversions.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestConversions.java?rev=799749&r1=799748&r2=799749&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestConversions.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestConversions.java Fri Jul 31 21:13:03 2009
@@ -17,12 +17,14 @@
*/
package org.apache.pig.test;
+import java.util.Iterator;
import java.util.Random;
import java.util.Map;
import java.io.IOException;
import org.apache.pig.builtin.PigStorage;
import org.apache.pig.test.utils.GenRandomData;
+import org.apache.pig.test.utils.TestHelper;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.data.DataBag;
@@ -44,6 +46,7 @@
PigStorage ps = new PigStorage();
Random r = new Random();
final int MAX = 10;
+
@Test
public void testBytesToInteger() throws IOException
@@ -153,7 +156,7 @@
for (int i = 0; i < MAX; i++) {
Tuple t = GenRandomData.genRandSmallBagTextTuple(r, 1, 100);
Tuple convertedTuple = ps.bytesToTuple(t.toString().getBytes());
- assertTrue(t.equals(convertedTuple));
+ assertTrue(TestHelper.tupleEquals(t, convertedTuple));
}
}
@@ -164,7 +167,7 @@
for (int i = 0; i < MAX; i++) {
DataBag b = GenRandomData.genRandFullTupTextDataBag(r,5,100);
DataBag convertedBag = ps.bytesToBag(b.toString().getBytes());
- assertTrue(b.equals(convertedBag));
+ assertTrue(TestHelper.bagEquals(b, convertedBag));
}
}
@@ -177,7 +180,7 @@
Map<String, Object> m = GenRandomData.genRandMap(r,5);
String expectedMapString = DataType.mapToString(m);
Map<String, Object> convertedMap = ps.bytesToMap(expectedMapString.getBytes());
- assertTrue(m.equals(convertedMap));
+ assertTrue(TestHelper.mapEquals(m, convertedMap));
}
}
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestStore.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestStore.java?rev=799749&r1=799748&r2=799749&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestStore.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestStore.java Fri Jul 31 21:13:03 2009
@@ -207,7 +207,7 @@
t.append(flds[8].compareTo("")!=0 ? ps.bytesToTuple(flds[8].getBytes()) : null);
t.append(flds[9].compareTo("")!=0 ? ps.bytesToCharArray(flds[9].getBytes()) : null);
- assertTrue(inputTuple.equals(t));
+ assertTrue(TestHelper.tupleEquals(inputTuple, t));
++size;
}
FileLocalizer.delete(fSpec.getFileName(), pc);
Added: hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java?rev=799749&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java Fri Jul 31 21:13:03 2009
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.test;
+
+import java.io.ByteArrayInputStream;
+import java.util.Map;
+
+import org.junit.Test;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataByteArray;
+import org.apache.pig.data.DefaultBagFactory;
+import org.apache.pig.data.DefaultTupleFactory;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.data.parser.TextDataParser;
+import org.apache.pig.data.parser.ParseException ;
+
+
+public class TestTextDataParser extends junit.framework.TestCase {
+
+ private final Log log = LogFactory.getLog(getClass());
+ private TupleFactory tupleFactory = DefaultTupleFactory.getInstance();
+ private BagFactory bagFactory = DefaultBagFactory.getInstance();
+
+
+ @Test
+ public void testInteger() throws Exception{
+ String myInteger = "1";
+ Integer i = (Integer)parseTextData(myInteger);
+ assertTrue(i.equals(1));
+ }
+
+ @Test
+ public void testLong() throws Exception{
+ String myLong = "1l";
+ Long l = (Long)parseTextData(myLong);
+ assertTrue(l.equals(1l));
+ }
+
+ @Test
+ public void testFloat() throws Exception{
+ String myFloat = "0.1f";
+ Float f = (Float)parseTextData(myFloat);
+ assertTrue(f.equals(0.1f));
+ }
+
+ @Test
+ public void testDouble() throws Exception{
+ String myDouble = "0.1";
+ Double d = (Double)parseTextData(myDouble);
+ assertTrue(d.equals(0.1));
+ }
+
+ @Test
+ public void testString() throws Exception{
+ String myString = "1a";
+ String s = (String)parseTextData(myString);
+ assertTrue(s.equals(myString));
+ }
+
+
+ //the value types of a map should always be a byte array
+ //irrespective of the actual type
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testMapStringValueType() throws Exception{
+ String myMap = "[key1#value1]";
+ Map<String, Object> map = (Map<String, Object>)parseTextData(myMap);
+ String key = map.keySet().iterator().next();
+ Object v = map.get("key1");
+ assertTrue(key.equals("key1"));
+ assertTrue(v instanceof DataByteArray);
+ String value = new String(((DataByteArray)v).get());
+ assertTrue(value.equals("value1"));
+ }
+
+ //the value types of a map should always be a byte array
+ //irrespective of the actual type
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testMapIntegerValueType() throws Exception{
+ String myMap = "[key1#1]";
+ Map<String, Object> map = (Map<String, Object>)parseTextData(myMap);
+ String key = map.keySet().iterator().next();
+ Object v = map.get("key1");
+ assertTrue(key.equals("key1"));
+ assertTrue(v instanceof DataByteArray);
+ String value = new String(((DataByteArray)v).get());
+ assertTrue(value.equals("1"));
+ }
+
+ //the value types of a map should always be a byte array
+ //irrespective of the actual type
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testMapLongValueType() throws Exception{
+ String myMap = "[key1#1l]";
+ Map<String, Object> map = (Map<String, Object>)parseTextData(myMap);
+ String key = map.keySet().iterator().next();
+ Object v = map.get("key1");
+ assertTrue(key.equals("key1"));
+ assertTrue(v instanceof DataByteArray);
+ String value = new String(((DataByteArray)v).get());
+ assertTrue(value.equals("1l"));
+ }
+
+ //the value types of a map should always be a byte array
+ //irrespective of the actual type
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testMapFloatValueType() throws Exception{
+ String myMap = "[key1#0.1f]";
+ Map<String, Object> map = (Map<String, Object>)parseTextData(myMap);
+ String key = map.keySet().iterator().next();
+ Object v = map.get("key1");
+ assertTrue(key.equals("key1"));
+ assertTrue(v instanceof DataByteArray);
+ String value = new String(((DataByteArray)v).get());
+ assertTrue(value.equals("0.1f"));
+ }
+
+ //the value types of a map should always be a byte array
+ //irrespective of the actual type
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testMapDoubleValueType() throws Exception{
+ String myMap = "[key1#0.1]";
+ Map<String, Object> map = (Map<String, Object>)parseTextData(myMap);
+ String key = map.keySet().iterator().next();
+ Object v = map.get("key1");
+ assertTrue(key.equals("key1"));
+ assertTrue(v instanceof DataByteArray);
+ String value = new String(((DataByteArray)v).get());
+ assertTrue(value.equals("0.1"));
+ }
+
+ @Test
+ public void testTuple() throws Exception{
+ String myTuple = "(1,a)";
+ Object o = parseTextData(myTuple);
+ assertTrue(o instanceof Tuple);
+ Tuple t = (Tuple)o;
+ Tuple expectedTuple = tupleFactory.newTuple(2);
+ expectedTuple.set(0, 1);
+ expectedTuple.set(1, "a");
+ assertTrue(t.equals(expectedTuple));
+ }
+
+ @Test
+ public void testBag() throws Exception{
+ String myBag = "{(1,a),(2,b)}";
+ Object o = parseTextData(myBag);
+ assertTrue(o instanceof DataBag);
+ DataBag b = (DataBag)o;
+ DataBag expectedBag = bagFactory.newDefaultBag();
+ Tuple expectedTuple = tupleFactory.newTuple(2);
+ expectedTuple.set(0, 1);
+ expectedTuple.set(1, "a");
+ expectedBag.add(expectedTuple);
+ expectedTuple = tupleFactory.newTuple(2);
+ expectedTuple.set(0, 2);
+ expectedTuple.set(1, "b");
+ expectedBag.add(expectedTuple);
+ assertTrue(b.equals(expectedBag));
+ }
+
+
+ private Object parseTextData(String pigConstantAsString) throws ParseException {
+ ByteArrayInputStream stream = new ByteArrayInputStream(pigConstantAsString.getBytes()) ;
+ TextDataParser textDataParser = new TextDataParser(stream) ;
+ return textDataParser.Datum();
+ }
+}
Modified: hadoop/pig/trunk/test/org/apache/pig/test/utils/TestHelper.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/utils/TestHelper.java?rev=799749&r1=799748&r2=799749&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/utils/TestHelper.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/utils/TestHelper.java Fri Jul 31 21:13:03 2009
@@ -19,6 +19,7 @@
import java.io.*;
import java.util.Iterator;
+import java.util.Map;
import java.util.Random;
import org.apache.pig.backend.executionengine.ExecException;
@@ -45,7 +46,7 @@
public static boolean bagContains(DataBag db, Tuple t) {
Iterator<Tuple> iter = db.iterator();
for (Tuple tuple : db) {
- if (tuple.compareTo(t) == 0)
+ if (tuple.compareTo(t) == 0 || tupleEquals(tuple, t))
return true;
}
return false;
@@ -240,4 +241,134 @@
ps.close();
return fp1 ;
}
+
+ //a quick way to check for map equality as the map value returned by PigStorage has byte array
+ public static boolean mapEquals(Map<String, Object> expectedMap, Map<String, Object> convertedMap) {
+ if(expectedMap == null) {
+ if(convertedMap != null) {
+ return false;
+ }
+ } else {
+ if (convertedMap == null) {
+ return false;
+ }
+ }
+
+ if(expectedMap.size() != convertedMap.size()) {
+ return false;
+ }
+
+ for(String key: expectedMap.keySet()) {
+ Object v = convertedMap.get(key);
+ String convertedValue = new String(((DataByteArray)v).get());
+ if(!expectedMap.get(key).equals(convertedValue)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @SuppressWarnings("unchecked")
+ public static boolean tupleEquals(Tuple expectedTuple, Tuple convertedTuple) {
+ if(expectedTuple == null) {
+ if(convertedTuple != null) {
+ return false;
+ }
+ } else {
+ if(convertedTuple == null) {
+ return false;
+ }
+ }
+
+ if(expectedTuple.size() != convertedTuple.size()) {
+ return false;
+ }
+
+ for(int i = 0; i < expectedTuple.size(); ++i) {
+ Object e ;
+ Object c ;
+
+ try {
+ e = expectedTuple.get(i);
+ c = convertedTuple.get(i);
+ } catch (Exception e1) {
+ return false;
+ }
+
+ if(e instanceof Map) {
+ Map<String, Object> eMap = (Map<String, Object>)e;
+ if(c instanceof Map) {
+ Map<String, Object> cMap = (Map<String, Object>)c;
+ if(!mapEquals(eMap, cMap)) {
+ return false;
+ }
+ } else {
+ return false;
+ }
+ } else if (e instanceof Tuple) {
+ if(c instanceof Tuple) {
+ if(!tupleEquals((Tuple)e, (Tuple)c)) {
+ return false;
+ }
+ } else {
+ return false;
+ }
+ } else if (e instanceof DataBag){
+ if(c instanceof DataBag) {
+ if(!bagEquals((DataBag)e, (DataBag)c)) {
+ return false;
+ }
+ } else {
+ return false;
+ }
+ } else {
+ if(e == null) {
+ if(c != null) {
+ return false;
+ }
+ } else {
+ if(c == null) {
+ return false;
+ } else {
+ if(!e.equals(c)) {
+ return false;
+ }
+ }
+ }
+ }
+ }
+
+ return true;
+ }
+
+ public static boolean bagEquals(DataBag expectedBag, DataBag convertedBag) {
+ if(expectedBag == null) {
+ if(convertedBag != null) {
+ return false;
+ }
+ } else {
+ if(convertedBag == null) {
+ return false;
+ }
+ }
+
+ if(expectedBag.size() != convertedBag.size()) {
+ return false;
+ }
+
+ Iterator<Tuple> expectedBagIterator = expectedBag.iterator();
+ Iterator<Tuple> convertedBagIterator = convertedBag.iterator();
+
+ while(expectedBagIterator.hasNext()) {
+ Tuple expectedBagTuple = expectedBagIterator.next();
+ Tuple convertedBagTuple = convertedBagIterator.next();
+ if(!tupleEquals(expectedBagTuple, convertedBagTuple)) {
+ return false;
+ }
+ }
+
+ return true;
+
+ }
+
}