You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2014/10/13 19:54:39 UTC
svn commit: r1631487 - in /hive/trunk/serde/src:
java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java
test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
Author: jdere
Date: Mon Oct 13 17:54:38 2014
New Revision: 1631487
URL: http://svn.apache.org/r1631487
Log:
HIVE-8115: Hive select query hang when fields contain map (Xiaobing Zhou via Jason Dere)
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java?rev=1631487&r1=1631486&r2=1631487&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java Mon Oct 13 17:54:38 2014
@@ -23,6 +23,8 @@ import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.io.Text;
@@ -36,6 +38,7 @@ import org.apache.hadoop.io.Text;
*/
public class LazyMap extends LazyNonPrimitive<LazyMapObjectInspector> {
+ public static final Log LOG = LogFactory.getLog(LazyMap.class);
/**
* Whether the data is already parsed or not.
*/
@@ -170,15 +173,19 @@ public class LazyMap extends LazyNonPrim
valueLength[mapSize] = elementByteEnd - (keyEnd[mapSize] + 1);
LazyPrimitive<?, ?> lazyKey = uncheckedGetKey(mapSize);
if (lazyKey == null) {
- continue;
- }
- Object key = lazyKey.getObject();
- if(!keySet.contains(key)) {
- mapSize++;
- keySet.add(key);
- } else {
+ LOG.warn("skipped empty entry or entry with empty key in the representation of column with MAP type.");
+ //reset keyInited[mapSize] flag, since it may be set to true in the case of previous empty entry
keyInited[mapSize] = false;
+ } else {
+ Object key = lazyKey.getObject();
+ if(!keySet.contains(key)) {
+ mapSize++;
+ keySet.add(key);
+ } else {
+ keyInited[mapSize] = false;
+ }
}
+
// reset keyValueSeparatorPosition
keyValueSeparatorPosition = -1;
elementByteBegin = elementByteEnd + 1;
Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java?rev=1631487&r1=1631486&r2=1631487&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java Mon Oct 13 17:54:38 2014
@@ -194,6 +194,113 @@ public class TestLazyArrayMapStruct exte
}
}
+ /*
+ * test LazyMap with bad entries, e.g., empty key or empty entries
+ * where '[' and ']' don't exist, only for notation purpose,
+ * STX with value of 2 as entry separator, ETX with 3 as key/value separator
+ * */
+ public void testLazyMapWithBadEntries() throws Throwable {
+ try {
+ {
+ // Map of String to String
+ Text nullSequence = new Text("");
+ ObjectInspector oi = LazyFactory.createLazyObjectInspector(
+ TypeInfoUtils.getTypeInfosFromTypeString("map<string,string>").get(
+ 0), new byte[] {'\2', '\3'}, 0, nullSequence,
+ false, (byte) 0);
+ LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi);
+
+ //read friendly string: ak[EXT]av[STX]bk[ETX]bv[STX]ck[ETX]cv[STX]dk[ETX]dv
+ byte[] data = new byte[] {
+ 'a', 'k', '\3', 'a', 'v',
+ '\02', 'b', 'k', '\3', 'b', 'v',
+ '\02', 'c', 'k', '\3', 'c', 'v',
+ '\02', 'd', 'k', '\3', 'd', 'v'};
+ TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
+
+ assertEquals(new Text("av"), ((LazyString) b
+ .getMapValueElement(new Text("ak"))).getWritableObject());
+ assertNull(b.getMapValueElement(new Text("-1")));
+ assertEquals(new Text("bv"), ((LazyString) b
+ .getMapValueElement(new Text("bk"))).getWritableObject());
+ assertEquals(new Text("cv"), ((LazyString) b
+ .getMapValueElement(new Text("ck"))).getWritableObject());
+ assertNull(b.getMapValueElement(new Text("-")));
+ assertEquals(new Text("dv"), ((LazyString) b
+ .getMapValueElement(new Text("dk"))).getWritableObject());
+ assertEquals(4, b.getMapSize());
+ }
+
+ {
+ // Map of String to String, LazyMap allows empty-string style key, e.g., {"" : null}
+ // or {"", ""}, but not null style key, e.g., {null:""}
+ Text nullSequence = new Text("");
+ ObjectInspector oi = LazyFactory.createLazyObjectInspector(
+ TypeInfoUtils.getTypeInfosFromTypeString("map<string,string>").get(
+ 0), new byte[] {'\2', '\3'}, 0, nullSequence,
+ false, (byte) 0);
+ LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi);
+
+ //read friendly string: [STX]ak[EXT]av[STX]bk[ETX]bv[STX]ck[ETX]cv[STX]dk[ETX]dv
+ byte[] data = new byte[] {
+ '\02', 'a', 'k', '\3', 'a', 'v',
+ '\02', 'b', 'k', '\3', 'b', 'v',
+ '\02', 'c', 'k', '\3', 'c', 'v',
+ '\02', 'd', 'k', '\3', 'd', 'v'};
+ TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
+
+ assertNull(b.getMapValueElement(new Text(""))); //{"" : null}
+ assertEquals(new Text("av"), ((LazyString) b
+ .getMapValueElement(new Text("ak"))).getWritableObject());
+ assertNull(b.getMapValueElement(new Text("-1")));
+ assertEquals(new Text("bv"), ((LazyString) b
+ .getMapValueElement(new Text("bk"))).getWritableObject());
+ assertEquals(new Text("cv"), ((LazyString) b
+ .getMapValueElement(new Text("ck"))).getWritableObject());
+ assertNull(b.getMapValueElement(new Text("-")));
+ assertEquals(new Text("dv"), ((LazyString) b
+ .getMapValueElement(new Text("dk"))).getWritableObject());
+ assertEquals(4, b.getMapSize());
+ }
+
+ {
+ // Map of String to String, LazyMap allows empty-string style key, e.g., {"" : null}
+ // or {"", ""}, but not null style key, e.g., {null:""}
+ Text nullSequence = new Text("");
+ ObjectInspector oi = LazyFactory.createLazyObjectInspector(
+ TypeInfoUtils.getTypeInfosFromTypeString("map<string,string>").get(
+ 0), new byte[] {'\2', '\3'}, 0, nullSequence,
+ false, (byte) 0);
+ LazyMap b = (LazyMap) LazyFactory.createLazyObject(oi);
+
+ //read friendly string: [ETX][STX]ak[EXT]av[STX]bk[ETX]bv[STX]ck[ETX]cv[STX]dk[ETX]dv
+ byte[] data = new byte[] {
+ '\03',
+ '\02', 'a', 'k', '\3', 'a', 'v',
+ '\02', 'b', 'k', '\3', 'b', 'v',
+ '\02', 'c', 'k', '\3', 'c', 'v',
+ '\02', 'd', 'k', '\3', 'd', 'v'};
+ TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
+
+ assertNull(b.getMapValueElement(new Text(""))); //{"" : null}
+ assertEquals(new Text("av"), ((LazyString) b
+ .getMapValueElement(new Text("ak"))).getWritableObject());
+ assertNull(b.getMapValueElement(new Text("-1")));
+ assertEquals(new Text("bv"), ((LazyString) b
+ .getMapValueElement(new Text("bk"))).getWritableObject());
+ assertEquals(new Text("cv"), ((LazyString) b
+ .getMapValueElement(new Text("ck"))).getWritableObject());
+ assertNull(b.getMapValueElement(new Text("-")));
+ assertEquals(new Text("dv"), ((LazyString) b
+ .getMapValueElement(new Text("dk"))).getWritableObject());
+ assertEquals(4, b.getMapSize());
+ }
+ } catch(Throwable e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
/**
* Test the LazyMap class.
*/