You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vi...@apache.org on 2014/03/24 20:12:23 UTC
svn commit: r1581006 - in /hive/branches/branch-0.13/serde/src:
java/org/apache/hadoop/hive/serde2/lazy/
java/org/apache/hadoop/hive/serde2/lazybinary/
test/org/apache/hadoop/hive/serde2/lazy/
Author: vikram
Date: Mon Mar 24 19:12:23 2014
New Revision: 1581006
URL: http://svn.apache.org/r1581006
Log:
Lazy maps are broken (LazyMap and LazyBinaryMap) (Prasanth J via Vikram Dixit, reviewed by Gunther)
Modified:
hive/branches/branch-0.13/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java
hive/branches/branch-0.13/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryPrimitive.java
hive/branches/branch-0.13/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
Modified: hive/branches/branch-0.13/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java?rev=1581006&r1=1581005&r2=1581006&view=diff
==============================================================================
--- hive/branches/branch-0.13/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java (original)
+++ hive/branches/branch-0.13/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java Mon Mar 24 19:12:23 2014
@@ -67,6 +67,23 @@ public abstract class LazyPrimitive<OI e
return isNull ? 0 : data.hashCode();
}
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof LazyPrimitive<?, ?>)) {
+ return false;
+ }
+
+ if (data == obj) {
+ return true;
+ }
+
+ if (data == null || obj == null) {
+ return false;
+ }
+
+ return data.equals(((LazyPrimitive<?, ?>) obj).getWritableObject());
+ }
+
public void logExceptionMessage(ByteArrayRef bytes, int start, int length, String dataType) {
try {
if(LOG.isDebugEnabled()) {
Modified: hive/branches/branch-0.13/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryPrimitive.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryPrimitive.java?rev=1581006&r1=1581005&r2=1581006&view=diff
==============================================================================
--- hive/branches/branch-0.13/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryPrimitive.java (original)
+++ hive/branches/branch-0.13/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryPrimitive.java Mon Mar 24 19:12:23 2014
@@ -65,4 +65,21 @@ public abstract class LazyBinaryPrimitiv
public int hashCode() {
return data == null ? 0 : data.hashCode();
}
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof LazyBinaryPrimitive<?, ?>)) {
+ return false;
+ }
+
+ if (data == obj) {
+ return true;
+ }
+
+ if (data == null || obj == null) {
+ return false;
+ }
+
+ return data.equals(((LazyBinaryPrimitive<?, ?>) obj).getWritableObject());
+ }
}
Modified: hive/branches/branch-0.13/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java?rev=1581006&r1=1581005&r2=1581006&view=diff
==============================================================================
--- hive/branches/branch-0.13/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java (original)
+++ hive/branches/branch-0.13/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java Mon Mar 24 19:12:23 2014
@@ -31,6 +31,9 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
+import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryFactory;
+import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryMap;
+import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
@@ -179,7 +182,7 @@ public class TestLazyArrayMapStruct exte
.getMapValueElement(new Text("8"))).getWritableObject());
assertNull(b.getMapValueElement(new Text("-")));
- assertEquals("{'2':'d\\tf','2':'d','-1':null,'0':'0','8':'abc'}"
+ assertEquals("{'2':'d\\tf','-1':null,'0':'0','8':'abc'}"
.replace('\'', '\"'), SerDeUtils.getJSONString(b, oi));
}
@@ -190,6 +193,67 @@ public class TestLazyArrayMapStruct exte
}
/**
+ * Test the LazyMap class.
+ */
+ public void testLazyMapWithDuplicateKeys() throws Throwable {
+ try {
+ {
+ // Map of Integer to String
+ Text nullSequence = new Text("\\N");
+ ObjectInspector oi = LazyFactory
+ .createLazyObjectInspector(TypeInfoUtils
+ .getTypeInfosFromTypeString("map<int,string>").get(0),
+ new byte[] {(byte) 1, (byte) 2}, 0, nullSequence, false,
+ (byte) 0);
+ LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi);
+ byte[] data = new byte[] {'2', 2, 'd', 'e', 'f', 1, '-', '1', 2, '\\',
+ 'N', 1, '0', 2, '0', 1, '2', 2, 'a', 'b', 'c'};
+ TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
+
+ assertEquals(new Text("def"), ((LazyString) b
+ .getMapValueElement(new IntWritable(2))).getWritableObject());
+ assertNull(b.getMapValueElement(new IntWritable(-1)));
+ assertEquals(new Text("0"), ((LazyString) b
+ .getMapValueElement(new IntWritable(0))).getWritableObject());
+ assertNull(b.getMapValueElement(new IntWritable(12345)));
+
+ assertEquals("{2:'def',-1:null,0:'0'}".replace('\'', '\"'),
+ SerDeUtils.getJSONString(b, oi));
+
+ assertEquals(3, b.getMap().size());
+ }
+
+ {
+ // Map of String to String
+ Text nullSequence = new Text("\\N");
+ ObjectInspector oi = LazyFactory.createLazyObjectInspector(
+ TypeInfoUtils.getTypeInfosFromTypeString("map<string,string>").get(
+ 0), new byte[] {(byte) '#', (byte) '\t'}, 0, nullSequence,
+ false, (byte) 0);
+ LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi);
+ byte[] data = new byte[] {'2', '\t', 'd', '\t', 'f', '#', '2', '\t',
+ 'd', '#', '-', '1', '#', '0', '\t', '0', '#', '2', '\t', 'a', 'b', 'c'};
+ TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
+
+ assertEquals(new Text("d\tf"), ((LazyString) b
+ .getMapValueElement(new Text("2"))).getWritableObject());
+ assertNull(b.getMapValueElement(new Text("-1")));
+ assertEquals(new Text("0"), ((LazyString) b
+ .getMapValueElement(new Text("0"))).getWritableObject());
+
+ assertEquals("{'2':'d\\tf','-1':null,'0':'0'}"
+ .replace('\'', '\"'), SerDeUtils.getJSONString(b, oi));
+
+ assertEquals(3, b.getMap().size());
+ }
+
+ } catch (Throwable e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ /**
* Test the LazyStruct class.
*/
public void testLazyStruct() throws Throwable {
@@ -243,7 +307,7 @@ public class TestLazyArrayMapStruct exte
data = new Text(": : : :");
TestLazyPrimitive.initLazyObject(o, data.getBytes(), 0, data
.getLength());
- assertEquals("{'a':null,'b':['',''],'c':{'':null,'':null},'d':':'}"
+ assertEquals("{'a':null,'b':['',''],'c':{'':null},'d':':'}"
.replace("'", "\""), SerDeUtils.getJSONString(o, oi));
data = new Text("= = = =");