You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by we...@apache.org on 2017/05/24 23:51:47 UTC
[13/54] [abbrv] hive git commit: HIVE-16207: Add support for Complex
Types in Fast SerDe (Teddy Choi, reviewed by Matt McCline)
http://git-wip-us.apache.org/repos/asf/hive/blob/d467e172/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java b/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java
index b369462..8356a5c 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hive.serde2.binarysortable;
+import java.io.IOException;
import java.util.ArrayList;
import java.io.EOFException;
import java.util.Arrays;
@@ -30,13 +31,16 @@ import org.apache.hadoop.hive.serde2.SerdeRandomRowSource;
import org.apache.hadoop.hive.serde2.VerifyFast;
import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableDeserializeRead;
import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableSerializeWrite;
+import org.apache.hadoop.hive.serde2.lazy.VerifyLazy;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.objectinspector.UnionObject;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Writable;
import junit.framework.TestCase;
+import org.junit.Assert;
public class TestBinarySortableFast extends TestCase {
@@ -48,11 +52,11 @@ public class TestBinarySortableFast extends TestCase {
boolean[] columnSortOrderIsDesc, byte[] columnNullMarker, byte[] columnNotNullMarker,
AbstractSerDe serde, StructObjectInspector rowOI,
AbstractSerDe serde_fewer, StructObjectInspector writeRowOI,
- boolean ascending, PrimitiveTypeInfo[] primitiveTypeInfos,
+ boolean ascending, TypeInfo[] typeInfos,
boolean useIncludeColumns, boolean doWriteFewerColumns, Random r) throws Throwable {
int rowCount = rows.length;
- int columnCount = primitiveTypeInfos.length;
+ int columnCount = typeInfos.length;
boolean[] columnsToInclude = null;
if (useIncludeColumns) {
@@ -83,10 +87,7 @@ public class TestBinarySortableFast extends TestCase {
int[] perFieldWriteLengths = new int[columnCount];
for (int index = 0; index < writeColumnCount; index++) {
-
- Writable writable = (Writable) row[index];
-
- VerifyFast.serializeWrite(binarySortableSerializeWrite, primitiveTypeInfos[index], writable);
+ VerifyFast.serializeWrite(binarySortableSerializeWrite, typeInfos[index], row[index]);
perFieldWriteLengths[index] = output.getLength();
}
perFieldWriteLengthsArray[i] = perFieldWriteLengths;
@@ -95,7 +96,8 @@ public class TestBinarySortableFast extends TestCase {
bytesWritable.set(output.getData(), 0, output.getLength());
serializeWriteBytes[i] = bytesWritable;
if (i > 0) {
- int compareResult = serializeWriteBytes[i - 1].compareTo(serializeWriteBytes[i]);
+ BytesWritable previousBytesWritable = serializeWriteBytes[i - 1];
+ int compareResult = previousBytesWritable.compareTo(bytesWritable);
if ((compareResult < 0 && !ascending)
|| (compareResult > 0 && ascending)) {
System.out.println("Test failed in "
@@ -117,7 +119,7 @@ public class TestBinarySortableFast extends TestCase {
Object[] row = rows[i];
BinarySortableDeserializeRead binarySortableDeserializeRead =
new BinarySortableDeserializeRead(
- primitiveTypeInfos,
+ typeInfos,
/* useExternalBuffer */ false,
columnSortOrderIsDesc,
columnNullMarker,
@@ -132,10 +134,9 @@ public class TestBinarySortableFast extends TestCase {
binarySortableDeserializeRead.skipNextField();
} else if (index >= writeColumnCount) {
// Should come back a null.
- VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead, primitiveTypeInfos[index], null);
+ VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead, typeInfos[index], null);
} else {
- Writable writable = (Writable) row[index];
- VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead, primitiveTypeInfos[index], writable);
+ verifyRead(binarySortableDeserializeRead, typeInfos[index], row[index]);
}
}
if (writeColumnCount == columnCount) {
@@ -147,7 +148,7 @@ public class TestBinarySortableFast extends TestCase {
*/
BinarySortableDeserializeRead binarySortableDeserializeRead2 =
new BinarySortableDeserializeRead(
- primitiveTypeInfos,
+ typeInfos,
/* useExternalBuffer */ false,
columnSortOrderIsDesc,
columnNullMarker,
@@ -157,22 +158,24 @@ public class TestBinarySortableFast extends TestCase {
bytesWritable.getBytes(), 0, bytesWritable.getLength() - 1); // One fewer byte.
for (int index = 0; index < writeColumnCount; index++) {
- Writable writable = (Writable) row[index];
if (index == writeColumnCount - 1) {
boolean threw = false;
try {
- VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead2, primitiveTypeInfos[index], writable);
+ verifyRead(binarySortableDeserializeRead2, typeInfos[index], row[index]);
} catch (EOFException e) {
// debugDetailedReadPositionString = binarySortableDeserializeRead2.getDetailedReadPositionString();
// debugStackTrace = e.getStackTrace();
threw = true;
}
- TestCase.assertTrue(threw);
+
+ if (!threw && row[index] != null) {
+ Assert.fail();
+ }
} else {
if (useIncludeColumns && !columnsToInclude[index]) {
binarySortableDeserializeRead2.skipNextField();
} else {
- VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead2, primitiveTypeInfos[index], writable);
+ verifyRead(binarySortableDeserializeRead2, typeInfos[index], row[index]);
}
}
}
@@ -270,7 +273,7 @@ public class TestBinarySortableFast extends TestCase {
"\nSerDe: " +
serDeFields.toString() +
"\nperFieldWriteLengths " + Arrays.toString(perFieldWriteLengthsArray[i]) +
- "\nprimitiveTypeInfos " + Arrays.toString(primitiveTypeInfos) +
+ "\nprimitiveTypeInfos " + Arrays.toString(typeInfos) +
"\nrow " + Arrays.toString(row));
}
}
@@ -282,7 +285,7 @@ public class TestBinarySortableFast extends TestCase {
Object[] row = rows[i];
BinarySortableDeserializeRead binarySortableDeserializeRead =
new BinarySortableDeserializeRead(
- primitiveTypeInfos,
+ typeInfos,
/* useExternalBuffer */ false,
columnSortOrderIsDesc,
columnNullMarker,
@@ -297,10 +300,9 @@ public class TestBinarySortableFast extends TestCase {
binarySortableDeserializeRead.skipNextField();
} else if (index >= writeColumnCount) {
// Should come back a null.
- VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead, primitiveTypeInfos[index], null);
+ verifyRead(binarySortableDeserializeRead, typeInfos[index], null);
} else {
- Writable writable = (Writable) row[index];
- VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead, primitiveTypeInfos[index], writable);
+ verifyRead(binarySortableDeserializeRead, typeInfos[index], row[index]);
}
}
if (writeColumnCount == columnCount) {
@@ -309,11 +311,44 @@ public class TestBinarySortableFast extends TestCase {
}
}
- private void testBinarySortableFastCase(int caseNum, boolean doNonRandomFill, Random r)
+ private void verifyRead(BinarySortableDeserializeRead binarySortableDeserializeRead,
+ TypeInfo typeInfo, Object expectedObject) throws IOException {
+ if (typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE) {
+ VerifyFast.verifyDeserializeRead(binarySortableDeserializeRead, typeInfo, expectedObject);
+ } else {
+ Object complexFieldObj = VerifyFast.deserializeReadComplexType(binarySortableDeserializeRead, typeInfo);
+ if (expectedObject == null) {
+ if (complexFieldObj != null) {
+ TestCase.fail("Field reports not null but object is null (class " + complexFieldObj.getClass().getName() +
+ ", " + complexFieldObj.toString() + ")");
+ }
+ } else {
+ if (complexFieldObj == null) {
+ // It's hard to distinguish a union with null from a null union.
+ if (expectedObject instanceof UnionObject) {
+ UnionObject expectedUnion = (UnionObject) expectedObject;
+ if (expectedUnion.getObject() == null) {
+ return;
+ }
+ }
+ TestCase.fail("Field reports null but object is not null (class " + expectedObject.getClass().getName() +
+ ", " + expectedObject.toString() + ")");
+ }
+ }
+ if (!VerifyLazy.lazyCompare(typeInfo, complexFieldObj, expectedObject)) {
+ TestCase.fail("Comparision failed typeInfo " + typeInfo.toString());
+ }
+ }
+ }
+
+ private void testBinarySortableFastCase(
+ int caseNum, boolean doNonRandomFill, Random r, SerdeRandomRowSource.SupportedTypes supportedTypes, int depth)
throws Throwable {
SerdeRandomRowSource source = new SerdeRandomRowSource();
- source.init(r);
+
+ // UNDONE: Until Fast BinarySortable supports complex types -- disable.
+ source.init(r, supportedTypes, depth);
int rowCount = 1000;
Object[][] rows = source.randomRows(rowCount);
@@ -327,8 +362,8 @@ public class TestBinarySortableFast extends TestCase {
StructObjectInspector rowStructObjectInspector = source.rowStructObjectInspector();
- PrimitiveTypeInfo[] primitiveTypeInfos = source.primitiveTypeInfos();
- int columnCount = primitiveTypeInfos.length;
+ TypeInfo[] typeInfos = source.typeInfos();
+ int columnCount = typeInfos.length;
int writeColumnCount = columnCount;
StructObjectInspector writeRowStructObjectInspector = rowStructObjectInspector;
@@ -385,14 +420,14 @@ public class TestBinarySortableFast extends TestCase {
columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker,
serde_ascending, rowStructObjectInspector,
serde_ascending_fewer, writeRowStructObjectInspector,
- /* ascending */ true, primitiveTypeInfos,
+ /* ascending */ true, typeInfos,
/* useIncludeColumns */ false, /* doWriteFewerColumns */ false, r);
testBinarySortableFast(source, rows,
columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker,
serde_ascending, rowStructObjectInspector,
serde_ascending_fewer, writeRowStructObjectInspector,
- /* ascending */ true, primitiveTypeInfos,
+ /* ascending */ true, typeInfos,
/* useIncludeColumns */ true, /* doWriteFewerColumns */ false, r);
if (doWriteFewerColumns) {
@@ -400,14 +435,14 @@ public class TestBinarySortableFast extends TestCase {
columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker,
serde_ascending, rowStructObjectInspector,
serde_ascending_fewer, writeRowStructObjectInspector,
- /* ascending */ true, primitiveTypeInfos,
+ /* ascending */ true, typeInfos,
/* useIncludeColumns */ false, /* doWriteFewerColumns */ true, r);
testBinarySortableFast(source, rows,
columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker,
serde_ascending, rowStructObjectInspector,
serde_ascending_fewer, writeRowStructObjectInspector,
- /* ascending */ true, primitiveTypeInfos,
+ /* ascending */ true, typeInfos,
/* useIncludeColumns */ true, /* doWriteFewerColumns */ true, r);
}
@@ -420,14 +455,14 @@ public class TestBinarySortableFast extends TestCase {
columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker,
serde_descending, rowStructObjectInspector,
serde_ascending_fewer, writeRowStructObjectInspector,
- /* ascending */ false, primitiveTypeInfos,
+ /* ascending */ false, typeInfos,
/* useIncludeColumns */ false, /* doWriteFewerColumns */ false, r);
testBinarySortableFast(source, rows,
columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker,
serde_descending, rowStructObjectInspector,
serde_ascending_fewer, writeRowStructObjectInspector,
- /* ascending */ false, primitiveTypeInfos,
+ /* ascending */ false, typeInfos,
/* useIncludeColumns */ true, /* doWriteFewerColumns */ false, r);
if (doWriteFewerColumns) {
@@ -435,27 +470,27 @@ public class TestBinarySortableFast extends TestCase {
columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker,
serde_descending, rowStructObjectInspector,
serde_descending_fewer, writeRowStructObjectInspector,
- /* ascending */ false, primitiveTypeInfos,
+ /* ascending */ false, typeInfos,
/* useIncludeColumns */ false, /* doWriteFewerColumns */ true, r);
testBinarySortableFast(source, rows,
columnSortOrderIsDesc, columnNullMarker, columnNotNullMarker,
serde_descending, rowStructObjectInspector,
serde_descending_fewer, writeRowStructObjectInspector,
- /* ascending */ false, primitiveTypeInfos,
+ /* ascending */ false, typeInfos,
/* useIncludeColumns */ true, /* doWriteFewerColumns */ true, r);
}
}
- public void testBinarySortableFast() throws Throwable {
+ public void testBinarySortableFast(SerdeRandomRowSource.SupportedTypes supportedTypes, int depth) throws Throwable {
try {
Random r = new Random(35790);
int caseNum = 0;
for (int i = 0; i < 10; i++) {
- testBinarySortableFastCase(caseNum, (i % 2 == 0), r);
+ testBinarySortableFastCase(caseNum, (i % 2 == 0), r, supportedTypes, depth);
caseNum++;
}
@@ -465,6 +500,18 @@ public class TestBinarySortableFast extends TestCase {
}
}
+ public void testBinarySortableFastPrimitive() throws Throwable {
+ testBinarySortableFast(SerdeRandomRowSource.SupportedTypes.PRIMITIVE, 0);
+ }
+
+ public void testBinarySortableFastComplexDepthOne() throws Throwable {
+ testBinarySortableFast(SerdeRandomRowSource.SupportedTypes.ALL_EXCEPT_MAP, 1);
+ }
+
+ public void testBinarySortableFastComplexDepthFour() throws Throwable {
+ testBinarySortableFast(SerdeRandomRowSource.SupportedTypes.ALL_EXCEPT_MAP, 4);
+ }
+
private static String displayBytes(byte[] bytes, int start, int length) {
StringBuilder sb = new StringBuilder();
for (int i = start; i < start + length; i++) {
http://git-wip-us.apache.org/repos/asf/hive/blob/d467e172/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java
index c857b42..6e6a4c2 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hive.serde2.lazy;
+import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Properties;
import java.util.Random;
@@ -33,10 +35,11 @@ import org.apache.hadoop.hive.serde2.lazy.fast.LazySimpleDeserializeRead;
import org.apache.hadoop.hive.serde2.lazy.fast.LazySimpleSerializeWrite;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.UnionObject;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
import junit.framework.TestCase;
@@ -46,12 +49,12 @@ public class TestLazySimpleFast extends TestCase {
SerdeRandomRowSource source, Object[][] rows,
LazySimpleSerDe serde, StructObjectInspector rowOI,
LazySimpleSerDe serde_fewer, StructObjectInspector writeRowOI,
- byte separator, LazySerDeParameters serdeParams, LazySerDeParameters serdeParams_fewer,
- PrimitiveTypeInfo[] primitiveTypeInfos,
+ LazySerDeParameters serdeParams, LazySerDeParameters serdeParams_fewer,
+ TypeInfo[] typeInfos,
boolean useIncludeColumns, boolean doWriteFewerColumns, Random r) throws Throwable {
int rowCount = rows.length;
- int columnCount = primitiveTypeInfos.length;
+ int columnCount = typeInfos.length;
boolean[] columnsToInclude = null;
if (useIncludeColumns) {
@@ -62,10 +65,10 @@ public class TestLazySimpleFast extends TestCase {
}
int writeColumnCount = columnCount;
- PrimitiveTypeInfo[] writePrimitiveTypeInfos = primitiveTypeInfos;
+ TypeInfo[] writeTypeInfos = typeInfos;
if (doWriteFewerColumns) {
writeColumnCount = writeRowOI.getAllStructFieldRefs().size();
- writePrimitiveTypeInfos = Arrays.copyOf(primitiveTypeInfos, writeColumnCount);
+ writeTypeInfos = Arrays.copyOf(typeInfos, writeColumnCount);
}
// Try to serialize
@@ -75,16 +78,12 @@ public class TestLazySimpleFast extends TestCase {
Output output = new Output();
LazySimpleSerializeWrite lazySimpleSerializeWrite =
- new LazySimpleSerializeWrite(columnCount,
- separator, serdeParams);
+ new LazySimpleSerializeWrite(columnCount, serdeParams);
lazySimpleSerializeWrite.set(output);
for (int index = 0; index < columnCount; index++) {
-
- Writable writable = (Writable) row[index];
-
- VerifyFast.serializeWrite(lazySimpleSerializeWrite, primitiveTypeInfos[index], writable);
+ VerifyFast.serializeWrite(lazySimpleSerializeWrite, typeInfos[index], row[index]);
}
BytesWritable bytesWritable = new BytesWritable();
@@ -97,29 +96,24 @@ public class TestLazySimpleFast extends TestCase {
Object[] row = rows[i];
LazySimpleDeserializeRead lazySimpleDeserializeRead =
new LazySimpleDeserializeRead(
- writePrimitiveTypeInfos,
+ writeTypeInfos,
/* useExternalBuffer */ false,
- separator, serdeParams);
+ serdeParams);
BytesWritable bytesWritable = serializeWriteBytes[i];
byte[] bytes = bytesWritable.getBytes();
int length = bytesWritable.getLength();
lazySimpleDeserializeRead.set(bytes, 0, length);
- char[] chars = new char[length];
- for (int c = 0; c < chars.length; c++) {
- chars[c] = (char) (bytes[c] & 0xFF);
- }
-
for (int index = 0; index < columnCount; index++) {
if (useIncludeColumns && !columnsToInclude[index]) {
lazySimpleDeserializeRead.skipNextField();
} else if (index >= writeColumnCount) {
// Should come back a null.
- VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, primitiveTypeInfos[index], null);
+ verifyReadNull(lazySimpleDeserializeRead, typeInfos[index]);
} else {
- Writable writable = (Writable) row[index];
- VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, primitiveTypeInfos[index], writable);
+ Object expectedObject = row[index];
+ verifyRead(lazySimpleDeserializeRead, typeInfos[index], expectedObject);
}
}
if (writeColumnCount == columnCount) {
@@ -128,28 +122,22 @@ public class TestLazySimpleFast extends TestCase {
}
// Try to deserialize using SerDe class our Writable row objects created by SerializeWrite.
- for (int i = 0; i < rowCount; i++) {
- BytesWritable bytesWritable = serializeWriteBytes[i];
+ for (int rowIndex = 0; rowIndex < rowCount; rowIndex++) {
+ BytesWritable bytesWritable = serializeWriteBytes[rowIndex];
LazyStruct lazySimpleStruct = (LazyStruct) serde.deserialize(bytesWritable);
- Object[] row = rows[i];
+ Object[] row = rows[rowIndex];
for (int index = 0; index < columnCount; index++) {
- PrimitiveTypeInfo primitiveTypeInfo = primitiveTypeInfos[index];
- Writable writable = (Writable) row[index];
- LazyPrimitive lazyPrimitive = (LazyPrimitive) lazySimpleStruct.getField(index);
- Object object;
- if (lazyPrimitive != null) {
- object = lazyPrimitive.getWritableObject();
- } else {
- object = null;
- }
- if (writable == null || object == null) {
- if (writable != null || object != null) {
+ TypeInfo typeInfo = typeInfos[index];
+ Object expectedObject = row[index];
+ Object object = lazySimpleStruct.getField(index);
+ if (expectedObject == null || object == null) {
+ if (expectedObject != null || object != null) {
fail("SerDe deserialized NULL column mismatch");
}
} else {
- if (!object.equals(writable)) {
+ if (!VerifyLazy.lazyCompare(typeInfo, object, expectedObject)) {
fail("SerDe deserialized value does not match");
}
}
@@ -185,9 +173,9 @@ public class TestLazySimpleFast extends TestCase {
LazySimpleDeserializeRead lazySimpleDeserializeRead =
new LazySimpleDeserializeRead(
- writePrimitiveTypeInfos,
+ writeTypeInfos,
/* useExternalBuffer */ false,
- separator, serdeParams);
+ serdeParams);
byte[] bytes = serdeBytes[i];
lazySimpleDeserializeRead.set(bytes, 0, bytes.length);
@@ -197,10 +185,10 @@ public class TestLazySimpleFast extends TestCase {
lazySimpleDeserializeRead.skipNextField();
} else if (index >= writeColumnCount) {
// Should come back a null.
- VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, primitiveTypeInfos[index], null);
+ verifyReadNull(lazySimpleDeserializeRead, typeInfos[index]);
} else {
- Writable writable = (Writable) row[index];
- VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, primitiveTypeInfos[index], writable);
+ Object expectedObject = row[index];
+ verifyRead(lazySimpleDeserializeRead, typeInfos[index], expectedObject);
}
}
if (writeColumnCount == columnCount) {
@@ -209,6 +197,46 @@ public class TestLazySimpleFast extends TestCase {
}
}
+ private void verifyReadNull(LazySimpleDeserializeRead lazySimpleDeserializeRead,
+ TypeInfo typeInfo) throws IOException {
+ if (typeInfo.getCategory() == Category.PRIMITIVE) {
+ VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, typeInfo, null);
+ } else {
+ Object complexFieldObj = VerifyFast.deserializeReadComplexType(lazySimpleDeserializeRead, typeInfo);
+ if (complexFieldObj != null) {
+ TestCase.fail("Field report not null but object is null");
+ }
+ }
+ }
+
+ private void verifyRead(LazySimpleDeserializeRead lazySimpleDeserializeRead,
+ TypeInfo typeInfo, Object expectedObject) throws IOException {
+ if (typeInfo.getCategory() == Category.PRIMITIVE) {
+ VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, typeInfo, expectedObject);
+ } else {
+ Object complexFieldObj = VerifyFast.deserializeReadComplexType(lazySimpleDeserializeRead, typeInfo);
+ if (expectedObject == null) {
+ if (complexFieldObj != null) {
+ TestCase.fail("Field reports not null but object is null (class " + complexFieldObj.getClass().getName() + ", " + complexFieldObj.toString() + ")");
+ }
+ } else {
+ if (complexFieldObj == null) {
+ // It's hard to distinguish a union with null from a null union.
+ if (expectedObject instanceof UnionObject) {
+ UnionObject expectedUnion = (UnionObject) expectedObject;
+ if (expectedUnion.getObject() == null) {
+ return;
+ }
+ }
+ TestCase.fail("Field reports null but object is not null (class " + expectedObject.getClass().getName() + ", " + expectedObject.toString() + ")");
+ }
+ }
+ if (!VerifyLazy.lazyCompare(typeInfo, complexFieldObj, expectedObject)) {
+ TestCase.fail("Comparision failed typeInfo " + typeInfo.toString());
+ }
+ }
+ }
+
private byte[] copyBytes(Text serialized) {
byte[] result = new byte[serialized.getLength()];
System.arraycopy(serialized.getBytes(), 0, result, 0, serialized.getLength());
@@ -238,19 +266,25 @@ public class TestLazySimpleFast extends TestCase {
return serDe;
}
- private LazySerDeParameters getSerDeParams(String fieldNames, String fieldTypes) throws SerDeException {
+ private LazySerDeParameters getSerDeParams(String fieldNames, String fieldTypes,
+ byte[] separators) throws SerDeException {
Configuration conf = new Configuration();
Properties tbl = createProperties(fieldNames, fieldTypes);
- return new LazySerDeParameters(conf, tbl, LazySimpleSerDe.class.getName());
+ LazySerDeParameters lazySerDeParams = new LazySerDeParameters(conf, tbl, LazySimpleSerDe.class.getName());
+ for (int i = 0; i < separators.length; i++) {
+ lazySerDeParams.setSeparator(i, separators[i]);
+ }
+ return lazySerDeParams;
}
- public void testLazySimpleFastCase(int caseNum, boolean doNonRandomFill, Random r)
+ public void testLazySimpleFastCase(
+ int caseNum, boolean doNonRandomFill, Random r, SerdeRandomRowSource.SupportedTypes supportedTypes, int depth)
throws Throwable {
SerdeRandomRowSource source = new SerdeRandomRowSource();
- source.init(r);
+ source.init(r, supportedTypes, depth);
- int rowCount = 1000;
+ int rowCount = 100;
Object[][] rows = source.randomRows(rowCount);
if (doNonRandomFill) {
@@ -259,8 +293,8 @@ public class TestLazySimpleFast extends TestCase {
StructObjectInspector rowStructObjectInspector = source.rowStructObjectInspector();
- PrimitiveTypeInfo[] primitiveTypeInfos = source.primitiveTypeInfos();
- int columnCount = primitiveTypeInfos.length;
+ TypeInfo[] typeInfos = source.typeInfos();
+ int columnCount = typeInfos.length;
int writeColumnCount = columnCount;
StructObjectInspector writeRowStructObjectInspector = rowStructObjectInspector;
@@ -277,8 +311,11 @@ public class TestLazySimpleFast extends TestCase {
String fieldNames = ObjectInspectorUtils.getFieldNames(rowStructObjectInspector);
String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowStructObjectInspector);
+ // Use different separator values.
+ byte[] separators = new byte[] {(byte) 9, (byte) 2, (byte) 3, (byte) 4, (byte) 5, (byte) 6, (byte) 7, (byte) 8};
+
LazySimpleSerDe serde = getSerDe(fieldNames, fieldTypes);
- LazySerDeParameters serdeParams = getSerDeParams(fieldNames, fieldTypes);
+ LazySerDeParameters serdeParams = getSerDeParams(fieldNames, fieldTypes, separators);
LazySimpleSerDe serde_fewer = null;
LazySerDeParameters serdeParams_fewer = null;
@@ -287,22 +324,22 @@ public class TestLazySimpleFast extends TestCase {
String partialFieldTypes = ObjectInspectorUtils.getFieldTypes(writeRowStructObjectInspector);
serde_fewer = getSerDe(fieldNames, fieldTypes);
- serdeParams_fewer = getSerDeParams(partialFieldNames, partialFieldTypes);
+ serdeParams_fewer = getSerDeParams(partialFieldNames, partialFieldTypes, separators);
}
- byte separator = (byte) '\t';
+
testLazySimpleFast(
source, rows,
serde, rowStructObjectInspector,
serde_fewer, writeRowStructObjectInspector,
- separator, serdeParams, serdeParams_fewer, primitiveTypeInfos,
+ serdeParams, serdeParams_fewer, typeInfos,
/* useIncludeColumns */ false, /* doWriteFewerColumns */ false, r);
testLazySimpleFast(
source, rows,
serde, rowStructObjectInspector,
serde_fewer, writeRowStructObjectInspector,
- separator, serdeParams, serdeParams_fewer, primitiveTypeInfos,
+ serdeParams, serdeParams_fewer, typeInfos,
/* useIncludeColumns */ true, /* doWriteFewerColumns */ false, r);
if (doWriteFewerColumns) {
@@ -310,26 +347,26 @@ public class TestLazySimpleFast extends TestCase {
source, rows,
serde, rowStructObjectInspector,
serde_fewer, writeRowStructObjectInspector,
- separator, serdeParams, serdeParams_fewer, primitiveTypeInfos,
+ serdeParams, serdeParams_fewer, typeInfos,
/* useIncludeColumns */ false, /* doWriteFewerColumns */ true, r);
testLazySimpleFast(
source, rows,
serde, rowStructObjectInspector,
serde_fewer, writeRowStructObjectInspector,
- separator, serdeParams, serdeParams_fewer, primitiveTypeInfos,
+ serdeParams, serdeParams_fewer, typeInfos,
/* useIncludeColumns */ true, /* doWriteFewerColumns */ true, r);
}
}
- public void testLazySimpleFast() throws Throwable {
+ public void testLazySimpleFast(SerdeRandomRowSource.SupportedTypes supportedTypes, int depth) throws Throwable {
try {
- Random r = new Random(35790);
+ Random r = new Random(8322);
int caseNum = 0;
- for (int i = 0; i < 10; i++) {
- testLazySimpleFastCase(caseNum, (i % 2 == 0), r);
+ for (int i = 0; i < 20; i++) {
+ testLazySimpleFastCase(caseNum, (i % 2 == 0), r, supportedTypes, depth);
caseNum++;
}
@@ -338,4 +375,16 @@ public class TestLazySimpleFast extends TestCase {
throw e;
}
}
+
+ public void testLazyBinarySimplePrimitive() throws Throwable {
+ testLazySimpleFast(SerdeRandomRowSource.SupportedTypes.PRIMITIVE, 0);
+ }
+
+ public void testLazyBinarySimpleComplexDepthOne() throws Throwable {
+ testLazySimpleFast(SerdeRandomRowSource.SupportedTypes.ALL, 1);
+ }
+
+ public void testLazyBinarySimpleComplexDepthFour() throws Throwable {
+ testLazySimpleFast(SerdeRandomRowSource.SupportedTypes.ALL, 4);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/d467e172/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java
index e62a80a..13c73be 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hive.serde2.lazybinary;
+import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Random;
@@ -27,11 +29,14 @@ import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.SerdeRandomRowSource;
import org.apache.hadoop.hive.serde2.VerifyFast;
import org.apache.hadoop.hive.serde2.binarysortable.MyTestClass;
+import org.apache.hadoop.hive.serde2.lazy.VerifyLazy;
import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinaryDeserializeRead;
import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinarySerializeWrite;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.objectinspector.UnionObject;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Writable;
@@ -41,11 +46,11 @@ public class TestLazyBinaryFast extends TestCase {
SerdeRandomRowSource source, Object[][] rows,
AbstractSerDe serde, StructObjectInspector rowOI,
AbstractSerDe serde_fewer, StructObjectInspector writeRowOI,
- PrimitiveTypeInfo[] primitiveTypeInfos,
+ TypeInfo[] typeInfos,
boolean useIncludeColumns, boolean doWriteFewerColumns, Random r) throws Throwable {
int rowCount = rows.length;
- int columnCount = primitiveTypeInfos.length;
+ int columnCount = typeInfos.length;
boolean[] columnsToInclude = null;
if (useIncludeColumns) {
@@ -56,10 +61,10 @@ public class TestLazyBinaryFast extends TestCase {
}
int writeColumnCount = columnCount;
- PrimitiveTypeInfo[] writePrimitiveTypeInfos = primitiveTypeInfos;
+ TypeInfo[] writeTypeInfos = typeInfos;
if (doWriteFewerColumns) {
writeColumnCount = writeRowOI.getAllStructFieldRefs().size();
- writePrimitiveTypeInfos = Arrays.copyOf(primitiveTypeInfos, writeColumnCount);
+ writeTypeInfos = Arrays.copyOf(typeInfos, writeColumnCount);
}
LazyBinarySerializeWrite lazyBinarySerializeWrite =
@@ -73,10 +78,7 @@ public class TestLazyBinaryFast extends TestCase {
lazyBinarySerializeWrite.set(output);
for (int index = 0; index < writeColumnCount; index++) {
-
- Writable writable = (Writable) row[index];
-
- VerifyFast.serializeWrite(lazyBinarySerializeWrite, primitiveTypeInfos[index], writable);
+ VerifyFast.serializeWrite(lazyBinarySerializeWrite, typeInfos[index], row[index]);
}
BytesWritable bytesWritable = new BytesWritable();
@@ -92,7 +94,7 @@ public class TestLazyBinaryFast extends TestCase {
// column.
LazyBinaryDeserializeRead lazyBinaryDeserializeRead =
new LazyBinaryDeserializeRead(
- writePrimitiveTypeInfos,
+ writeTypeInfos,
/* useExternalBuffer */ false);
BytesWritable bytesWritable = serializeWriteBytes[i];
@@ -103,10 +105,9 @@ public class TestLazyBinaryFast extends TestCase {
lazyBinaryDeserializeRead.skipNextField();
} else if (index >= writeColumnCount) {
// Should come back a null.
- VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, primitiveTypeInfos[index], null);
+ VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, typeInfos[index], null);
} else {
- Writable writable = (Writable) row[index];
- VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, primitiveTypeInfos[index], writable);
+ verifyRead(lazyBinaryDeserializeRead, typeInfos[index], row[index]);
}
}
if (writeColumnCount == columnCount) {
@@ -127,15 +128,14 @@ public class TestLazyBinaryFast extends TestCase {
Object[] row = rows[i];
for (int index = 0; index < writeColumnCount; index++) {
- PrimitiveTypeInfo primitiveTypeInfo = primitiveTypeInfos[index];
- Writable writable = (Writable) row[index];
+ TypeInfo typeInfo = typeInfos[index];
Object object = lazyBinaryStruct.getField(index);
- if (writable == null || object == null) {
- if (writable != null || object != null) {
+ if (row[index] == null || object == null) {
+ if (row[index] != null || object != null) {
fail("SerDe deserialized NULL column mismatch");
}
} else {
- if (!object.equals(writable)) {
+ if (!VerifyLazy.lazyCompare(typeInfo, object, row[index])) {
fail("SerDe deserialized value does not match");
}
}
@@ -172,10 +172,10 @@ public class TestLazyBinaryFast extends TestCase {
if (bytes1.length != bytes2.length) {
fail("SerializeWrite length " + bytes2.length + " and " +
"SerDe serialization length " + bytes1.length +
- " do not match (" + Arrays.toString(primitiveTypeInfos) + ")");
+ " do not match (" + Arrays.toString(typeInfos) + ")");
}
if (!Arrays.equals(bytes1, bytes2)) {
- fail("SerializeWrite and SerDe serialization does not match (" + Arrays.toString(primitiveTypeInfos) + ")");
+ fail("SerializeWrite and SerDe serialization does not match (" + Arrays.toString(typeInfos) + ")");
}
serdeBytes[i] = bytesWritable;
}
@@ -187,7 +187,7 @@ public class TestLazyBinaryFast extends TestCase {
// When doWriteFewerColumns, try to read more fields than exist in buffer.
LazyBinaryDeserializeRead lazyBinaryDeserializeRead =
new LazyBinaryDeserializeRead(
- primitiveTypeInfos,
+ typeInfos,
/* useExternalBuffer */ false);
BytesWritable bytesWritable = serdeBytes[i];
@@ -198,10 +198,9 @@ public class TestLazyBinaryFast extends TestCase {
lazyBinaryDeserializeRead.skipNextField();
} else if (index >= writeColumnCount) {
// Should come back a null.
- VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, primitiveTypeInfos[index], null);
+ VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, typeInfos[index], null);
} else {
- Writable writable = (Writable) row[index];
- VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, primitiveTypeInfos[index], writable);
+ verifyRead(lazyBinaryDeserializeRead, typeInfos[index], row[index]);
}
}
if (writeColumnCount == columnCount) {
@@ -210,12 +209,45 @@ public class TestLazyBinaryFast extends TestCase {
}
}
- public void testLazyBinaryFastCase(int caseNum, boolean doNonRandomFill, Random r) throws Throwable {
+ private void verifyRead(LazyBinaryDeserializeRead lazyBinaryDeserializeRead,
+ TypeInfo typeInfo, Object expectedObject) throws IOException {
+ if (typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE) {
+ VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, typeInfo, expectedObject);
+ } else {
+ Object complexFieldObj = VerifyFast.deserializeReadComplexType(lazyBinaryDeserializeRead, typeInfo);
+ if (expectedObject == null) {
+ if (complexFieldObj != null) {
+ TestCase.fail("Field reports not null but object is null (class " + complexFieldObj.getClass().getName() +
+ ", " + complexFieldObj.toString() + ")");
+ }
+ } else {
+ if (complexFieldObj == null) {
+ // It's hard to distinguish a union with null from a null union.
+ if (expectedObject instanceof UnionObject) {
+ UnionObject expectedUnion = (UnionObject) expectedObject;
+ if (expectedUnion.getObject() == null) {
+ return;
+ }
+ }
+ TestCase.fail("Field reports null but object is not null (class " + expectedObject.getClass().getName() +
+ ", " + expectedObject.toString() + ")");
+ }
+ }
+ if (!VerifyLazy.lazyCompare(typeInfo, complexFieldObj, expectedObject)) {
+ TestCase.fail("Comparision failed typeInfo " + typeInfo.toString());
+ }
+ }
+ }
+
+ public void testLazyBinaryFastCase(
+ int caseNum, boolean doNonRandomFill, Random r, SerdeRandomRowSource.SupportedTypes supportedTypes, int depth)
+ throws Throwable {
SerdeRandomRowSource source = new SerdeRandomRowSource();
- source.init(r);
- int rowCount = 1000;
+ source.init(r, supportedTypes, depth);
+
+ int rowCount = 100;
Object[][] rows = source.randomRows(rowCount);
if (doNonRandomFill) {
@@ -224,8 +256,8 @@ public class TestLazyBinaryFast extends TestCase {
StructObjectInspector rowStructObjectInspector = source.rowStructObjectInspector();
- PrimitiveTypeInfo[] primitiveTypeInfos = source.primitiveTypeInfos();
- int columnCount = primitiveTypeInfos.length;
+ TypeInfo[] typeInfos = source.typeInfos();
+ int columnCount = typeInfos.length;
int writeColumnCount = columnCount;
StructObjectInspector writeRowStructObjectInspector = rowStructObjectInspector;
@@ -256,14 +288,14 @@ public class TestLazyBinaryFast extends TestCase {
source, rows,
serde, rowStructObjectInspector,
serde_fewer, writeRowStructObjectInspector,
- primitiveTypeInfos,
+ typeInfos,
/* useIncludeColumns */ false, /* doWriteFewerColumns */ false, r);
testLazyBinaryFast(
source, rows,
serde, rowStructObjectInspector,
serde_fewer, writeRowStructObjectInspector,
- primitiveTypeInfos,
+ typeInfos,
/* useIncludeColumns */ true, /* doWriteFewerColumns */ false, r);
/*
@@ -286,14 +318,13 @@ public class TestLazyBinaryFast extends TestCase {
// }
}
- public void testLazyBinaryFast() throws Throwable {
-
+ private void testLazyBinaryFast(SerdeRandomRowSource.SupportedTypes supportedTypes, int depth) throws Throwable {
try {
- Random r = new Random(35790);
+ Random r = new Random(9983);
int caseNum = 0;
for (int i = 0; i < 10; i++) {
- testLazyBinaryFastCase(caseNum, (i % 2 == 0), r);
+ testLazyBinaryFastCase(caseNum, (i % 2 == 0), r, supportedTypes, depth);
caseNum++;
}
@@ -302,4 +333,16 @@ public class TestLazyBinaryFast extends TestCase {
throw e;
}
}
+
+ public void testLazyBinaryFastPrimitive() throws Throwable {
+ testLazyBinaryFast(SerdeRandomRowSource.SupportedTypes.PRIMITIVE, 0);
+ }
+
+ public void testLazyBinaryFastComplexDepthOne() throws Throwable {
+ testLazyBinaryFast(SerdeRandomRowSource.SupportedTypes.ALL, 1);
+ }
+
+ public void testLazyBinaryFastComplexDepthFour() throws Throwable {
+ testLazyBinaryFast(SerdeRandomRowSource.SupportedTypes.ALL, 4);
+ }
}
\ No newline at end of file