You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2015/04/14 03:59:31 UTC
svn commit: r1673337 [10/10] - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/common/type/
common/src/java/org/apache/hive/common/util/ itests/src/test/resources/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql...
Added: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java?rev=1673337&view=auto
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java (added)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java Tue Apr 14 01:59:29 2015
@@ -0,0 +1,261 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.lazy;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.ByteStream.Output;
+import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.SerDeUtils;
+import org.apache.hadoop.hive.serde2.VerifyFast;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo;
+import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableDeserializeRead;
+import org.apache.hadoop.hive.serde2.lazy.fast.LazySimpleDeserializeRead;
+import org.apache.hadoop.hive.serde2.lazy.fast.LazySimpleSerializeWrite;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+
+public class TestLazySimpleFast extends TestCase {
+
+ private void testLazySimpleFast(MyTestPrimitiveClass[] myTestPrimitiveClasses, LazySimpleSerDe[] serdes,
+ StructObjectInspector[] rowOIs, byte separator, LazySerDeParameters[] serdeParams,
+ PrimitiveTypeInfo[][] primitiveTypeInfosArray) throws Throwable {
+
+
+ // Try to serialize
+ BytesWritable serializeWriteBytes[] = new BytesWritable[myTestPrimitiveClasses.length];
+ for (int i = 0; i < myTestPrimitiveClasses.length; i++) {
+ MyTestPrimitiveClass t = myTestPrimitiveClasses[i];
+ Output output = new Output();
+
+ LazySimpleSerializeWrite lazySimpleSerializeWrite =
+ new LazySimpleSerializeWrite(MyTestPrimitiveClass.primitiveCount,
+ separator, serdeParams[i]);
+
+ lazySimpleSerializeWrite.set(output);
+
+ for (int index = 0; index < MyTestPrimitiveClass.primitiveCount; index++) {
+ Object object = t.getPrimitiveObject(index);
+ PrimitiveCategory primitiveCategory = t.getPrimitiveCategory(index);
+ VerifyFast.serializeWrite(lazySimpleSerializeWrite, primitiveCategory, object);
+ }
+
+ BytesWritable bytesWritable = new BytesWritable();
+ bytesWritable.set(output.getData(), 0, output.getLength());
+ serializeWriteBytes[i] = bytesWritable;
+ }
+
+ // Try to deserialize
+ for (int i = 0; i < myTestPrimitiveClasses.length; i++) {
+ MyTestPrimitiveClass t = myTestPrimitiveClasses[i];
+ PrimitiveTypeInfo[] primitiveTypeInfos = primitiveTypeInfosArray[i];
+ LazySimpleDeserializeRead lazySimpleDeserializeRead =
+ new LazySimpleDeserializeRead(primitiveTypeInfos,
+ separator, serdeParams[i]);
+
+ BytesWritable bytesWritable = serializeWriteBytes[i];
+ byte[] bytes = bytesWritable.getBytes();
+ int length = bytesWritable.getLength();
+ lazySimpleDeserializeRead.set(bytes, 0, length);
+
+ char[] chars = new char[length];
+ for (int c = 0; c < chars.length; c++) {
+ chars[c] = (char) (bytes[c] & 0xFF);
+ }
+
+ for (int index = 0; index < MyTestPrimitiveClass.primitiveCount; index++) {
+ Object object = t.getPrimitiveObject(index);
+ PrimitiveCategory primitiveCategory = t.getPrimitiveCategory(index);
+ VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, primitiveTypeInfos[index], object);
+ }
+ lazySimpleDeserializeRead.extraFieldsCheck();
+ TestCase.assertTrue(!lazySimpleDeserializeRead.readBeyondConfiguredFieldsWarned());
+ TestCase.assertTrue(!lazySimpleDeserializeRead.readBeyondBufferRangeWarned());
+ TestCase.assertTrue(!lazySimpleDeserializeRead.bufferRangeHasExtraDataWarned());
+ }
+
+ // Try to deserialize using SerDe class our Writable row objects created by SerializeWrite.
+ for (int i = 0; i < myTestPrimitiveClasses.length; i++) {
+ BytesWritable bytesWritable = serializeWriteBytes[i];
+ LazyStruct lazySimpleStruct = (LazyStruct) serdes[i].deserialize(bytesWritable);
+
+ MyTestPrimitiveClass t = myTestPrimitiveClasses[i];
+ PrimitiveTypeInfo[] primitiveTypeInfos = primitiveTypeInfosArray[i];
+
+ for (int index = 0; index < MyTestPrimitiveClass.primitiveCount; index++) {
+ PrimitiveTypeInfo primitiveTypeInfo = primitiveTypeInfos[index];
+ Object expected = t.getPrimitiveWritableObject(index, primitiveTypeInfo);
+ LazyPrimitive lazyPrimitive = (LazyPrimitive) lazySimpleStruct.getField(index);
+ Object object;
+ if (lazyPrimitive != null) {
+ object = lazyPrimitive.getWritableObject();
+ } else {
+ object = null;
+ }
+ if (expected == null || object == null) {
+ if (expected != null || object != null) {
+ fail("SerDe deserialized NULL column mismatch");
+ }
+ } else {
+ if (!object.equals(expected)) {
+ fail("SerDe deserialized value does not match");
+ }
+ }
+ }
+ }
+
+ // One Writable per row.
+ byte[][] serdeBytes = new byte[myTestPrimitiveClasses.length][];
+
+ // Serialize using the SerDe, then below deserialize using DeserializeRead.
+ Object[] row = new Object[MyTestPrimitiveClass.primitiveCount];
+ for (int i = 0; i < myTestPrimitiveClasses.length; i++) {
+ MyTestPrimitiveClass t = myTestPrimitiveClasses[i];
+ PrimitiveTypeInfo[] primitiveTypeInfos = primitiveTypeInfosArray[i];
+
+ // LazySimple seems to work better with an row object array instead of a Java object...
+ for (int index = 0; index < MyTestPrimitiveClass.primitiveCount; index++) {
+ Object object = t.getPrimitiveWritableObject(index, primitiveTypeInfos[index]);
+ row[index] = object;
+ }
+
+ Text serialized = (Text) serdes[i].serialize(row, rowOIs[i]);
+ byte[] bytes1 = Arrays.copyOfRange(serialized.getBytes(), 0, serialized.getLength());
+
+ byte[] bytes2 = Arrays.copyOfRange(serializeWriteBytes[i].getBytes(), 0, serializeWriteBytes[i].getLength());
+ if (!Arrays.equals(bytes1, bytes2)) {
+ fail("SerializeWrite and SerDe serialization does not match");
+ }
+ serdeBytes[i] = serialized.copyBytes();
+ }
+
+ // Try to deserialize using DeserializeRead our Writable row objects created by SerDe.
+ for (int i = 0; i < myTestPrimitiveClasses.length; i++) {
+ MyTestPrimitiveClass t = myTestPrimitiveClasses[i];
+ PrimitiveTypeInfo[] primitiveTypeInfos = primitiveTypeInfosArray[i];
+ LazySimpleDeserializeRead lazySimpleDeserializeRead =
+ new LazySimpleDeserializeRead(primitiveTypeInfos,
+ separator, serdeParams[i]);
+
+ byte[] bytes = serdeBytes[i];
+ lazySimpleDeserializeRead.set(bytes, 0, bytes.length);
+
+ for (int index = 0; index < MyTestPrimitiveClass.primitiveCount; index++) {
+ Object object = t.getPrimitiveObject(index);
+ VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, primitiveTypeInfos[index], object);
+ }
+ lazySimpleDeserializeRead.extraFieldsCheck();
+ TestCase.assertTrue(!lazySimpleDeserializeRead.readBeyondConfiguredFieldsWarned());
+ TestCase.assertTrue(!lazySimpleDeserializeRead.readBeyondBufferRangeWarned());
+ TestCase.assertTrue(!lazySimpleDeserializeRead.bufferRangeHasExtraDataWarned());
+ }
+ }
+
+ private Properties createProperties(String fieldNames, String fieldTypes) {
+ Properties tbl = new Properties();
+
+ // Set the configuration parameters
+ tbl.setProperty(serdeConstants.SERIALIZATION_FORMAT, "9");
+
+ tbl.setProperty("columns", fieldNames);
+ tbl.setProperty("columns.types", fieldTypes);
+
+ tbl.setProperty(serdeConstants.SERIALIZATION_NULL_FORMAT, "NULL");
+
+ return tbl;
+ }
+
+ private LazySimpleSerDe getSerDe(String fieldNames, String fieldTypes) throws SerDeException {
+ // Create the SerDe
+ LazySimpleSerDe serDe = new LazySimpleSerDe();
+ Configuration conf = new Configuration();
+ Properties tbl = createProperties(fieldNames, fieldTypes);
+ SerDeUtils.initializeSerDe(serDe, conf, tbl, null);
+ return serDe;
+ }
+
+ private LazySerDeParameters getSerDeParams(String fieldNames, String fieldTypes) throws SerDeException {
+ Configuration conf = new Configuration();
+ Properties tbl = createProperties(fieldNames, fieldTypes);
+ return new LazySerDeParameters(conf, tbl, LazySimpleSerDe.class.getName());
+ }
+
+ public void testLazySimpleFast() throws Throwable {
+ try {
+
+ int num = 1000;
+ Random r = new Random(1234);
+ MyTestPrimitiveClass[] rows = new MyTestPrimitiveClass[num];
+ PrimitiveTypeInfo[][] primitiveTypeInfosArray = new PrimitiveTypeInfo[num][];
+ for (int i = 0; i < num; i++) {
+ int randField = r.nextInt(MyTestPrimitiveClass.primitiveCount);
+ MyTestPrimitiveClass t = new MyTestPrimitiveClass();
+ int field = 0;
+ ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
+ t.randomFill(r, randField, field, extraTypeInfo);
+ PrimitiveTypeInfo[] primitiveTypeInfos = MyTestPrimitiveClass.getPrimitiveTypeInfos(extraTypeInfo);
+ rows[i] = t;
+ primitiveTypeInfosArray[i] = primitiveTypeInfos;
+ }
+
+ // To get the specific type information for CHAR and VARCHAR, seems like we need an
+ // inspector and SerDe per row...
+ StructObjectInspector[] rowOIs = new StructObjectInspector[num];
+ LazySimpleSerDe[] serdes = new LazySimpleSerDe[num];
+ LazySerDeParameters[] serdeParams = new LazySerDeParameters[num];
+ for (int i = 0; i < num; i++) {
+ MyTestPrimitiveClass t = rows[i];
+
+ StructObjectInspector rowOI = t.getRowInspector(primitiveTypeInfosArray[i]);
+
+ String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI);
+ String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI);
+
+ rowOIs[i] = rowOI;
+ serdes[i] = getSerDe(fieldNames, fieldTypes);
+ serdeParams[i] = getSerDeParams(fieldNames, fieldTypes);
+ }
+
+ byte separator = (byte) '\t';
+ testLazySimpleFast(rows, serdes, rowOIs, separator, serdeParams, primitiveTypeInfosArray);
+ } catch (Throwable e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+}
\ No newline at end of file
Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassBigger.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassBigger.java?rev=1673337&r1=1673336&r2=1673337&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassBigger.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassBigger.java Tue Apr 14 01:59:29 2015
@@ -18,49 +18,119 @@
package org.apache.hadoop.hive.serde2.lazybinary;
import java.sql.Date;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Random;
+import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestClass;
import org.apache.hadoop.hive.serde2.binarysortable.MyTestInnerStruct;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass;
+import org.apache.hadoop.hive.serde2.binarysortable.TestBinarySortableSerDe;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo;
/**
* MyTestClassBigger.
*
*/
public class MyTestClassBigger {
- Byte myByte;
- Short myShort;
- Integer myInt;
- Long myLong;
- Float myFloat;
- Double myDouble;
- String myString;
- HiveDecimal myDecimal;
- Date myDate;
- MyTestInnerStruct myStruct;
- List<Integer> myList;
- byte[] myBA;
+
+ // The primitives.
+ public Boolean myBool;
+ public Byte myByte;
+ public Short myShort;
+ public Integer myInt;
+ public Long myLong;
+ public Float myFloat;
+ public Double myDouble;
+ public String myString;
+ public HiveChar myHiveChar;
+ public HiveVarchar myHiveVarchar;
+ public byte[] myBinary;
+ public HiveDecimal myDecimal;
+ public Date myDate;
+ public Timestamp myTimestamp;
+ public HiveIntervalYearMonth myIntervalYearMonth;
+ public HiveIntervalDayTime myIntervalDayTime;
+
+
+ // Add more complex types.
+ public MyTestInnerStruct myStruct;
+ public List<Integer> myList;
+
+ // Bigger addition.
Map<String, List<MyTestInnerStruct>> myMap;
+ public final static int mapPos = 18;
+
public MyTestClassBigger() {
}
- public MyTestClassBigger(Byte b, Short s, Integer i, Long l, Float f,
- Double d, String st, HiveDecimal bd, Date date, MyTestInnerStruct is, List<Integer> li,
- byte[] ba, Map<String, List<MyTestInnerStruct>> mp) {
- myByte = b;
- myShort = s;
- myInt = i;
- myLong = l;
- myFloat = f;
- myDouble = d;
- myString = st;
- myDecimal = bd;
- myDate = date;
- myStruct = is;
- myList = li;
- myBA = ba;
- myMap = mp;
+ public final static int biggerCount = 19;
+
+ public int randomFill(Random r, ExtraTypeInfo extraTypeInfo) {
+ int randField = r.nextInt(biggerCount);
+ int field = 0;
+ myBool = (randField == field++) ? null : (r.nextInt(1) == 1);
+ myByte = (randField == field++) ? null : Byte.valueOf((byte) r.nextInt());
+ myShort = (randField == field++) ? null : Short.valueOf((short) r.nextInt());
+ myInt = (randField == field++) ? null : Integer.valueOf(r.nextInt());
+ myLong = (randField == field++) ? null : Long.valueOf(r.nextLong());
+ myFloat = (randField == field++) ? null : Float
+ .valueOf(r.nextFloat() * 10 - 5);
+ myDouble = (randField == field++) ? null : Double
+ .valueOf(r.nextDouble() * 10 - 5);
+ myString = (randField == field++) ? null : MyTestPrimitiveClass.getRandString(r);
+ myHiveChar = (randField == field++) ? null : MyTestPrimitiveClass.getRandHiveChar(r, extraTypeInfo);
+ myHiveVarchar = (randField == field++) ? null : MyTestPrimitiveClass.getRandHiveVarchar(r, extraTypeInfo);
+ myBinary = MyTestPrimitiveClass.getRandBinary(r, r.nextInt(1000));
+ myDecimal = (randField == field++) ? null : MyTestPrimitiveClass.getRandHiveDecimal(r, extraTypeInfo);
+ myDate = (randField == field++) ? null : MyTestPrimitiveClass.getRandDate(r);
+ myTimestamp = (randField == field++) ? null : MyTestPrimitiveClass.getRandTimestamp(r);
+ myIntervalYearMonth = (randField == field++) ? null : MyTestPrimitiveClass.getRandIntervalYearMonth(r);
+ myIntervalDayTime = (randField == field++) ? null : MyTestPrimitiveClass.getRandIntervalDayTime(r);
+
+ myStruct = (randField == field++) ? null : new MyTestInnerStruct(
+ r.nextInt(5) - 2, r.nextInt(5) - 2);
+ myList = (randField == field++) ? null : MyTestClass.getRandIntegerArray(r);
+
+ Map<String, List<MyTestInnerStruct>> mp = new HashMap<String, List<MyTestInnerStruct>>();
+ String key = MyTestPrimitiveClass.getRandString(r);
+ List<MyTestInnerStruct> value = randField > 9 ? null
+ : getRandStructArray(r);
+ mp.put(key, value);
+ String key1 = MyTestPrimitiveClass.getRandString(r);
+ mp.put(key1, null);
+ String key2 = MyTestPrimitiveClass.getRandString(r);
+ List<MyTestInnerStruct> value2 = getRandStructArray(r);
+ mp.put(key2, value2);
+ myMap = mp;
+ return field;
}
+
+ /**
+ * Generate a random struct array.
+ *
+ * @param r
+ * random number generator
+ * @return an struct array
+ */
+ static List<MyTestInnerStruct> getRandStructArray(Random r) {
+ int length = r.nextInt(10);
+ ArrayList<MyTestInnerStruct> result = new ArrayList<MyTestInnerStruct>(
+ length);
+ for (int i = 0; i < length; i++) {
+ MyTestInnerStruct ti = new MyTestInnerStruct(r.nextInt(), r.nextInt());
+ result.add(ti);
+ }
+ return result;
+ }
+
}
Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassSmaller.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassSmaller.java?rev=1673337&r1=1673336&r2=1673337&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassSmaller.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassSmaller.java Tue Apr 14 01:59:29 2015
@@ -18,36 +18,68 @@
package org.apache.hadoop.hive.serde2.lazybinary;
import java.sql.Date;
+import java.sql.Timestamp;
+import java.util.Random;
+import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestClass;
import org.apache.hadoop.hive.serde2.binarysortable.MyTestInnerStruct;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo;
public class MyTestClassSmaller {
- Byte myByte;
- Short myShort;
- Integer myInt;
- Long myLong;
- Float myFloat;
- Double myDouble;
- String myString;
- HiveDecimal myDecimal;
- Date myDate;
+
+ public Boolean myBool;
+ public Byte myByte;
+ public Short myShort;
+ public Integer myInt;
+ public Long myLong;
+ public Float myFloat;
+ public Double myDouble;
+ public String myString;
+ public HiveChar myHiveChar;
+ public HiveVarchar myHiveVarchar;
+ public byte[] myBinary;
+ public HiveDecimal myDecimal;
+ public Date myDate;
+ public Timestamp myTimestamp;
+ public HiveIntervalYearMonth myIntervalYearMonth;
+ public HiveIntervalDayTime myIntervalDayTime;
+
MyTestInnerStruct myStruct;
- public MyTestClassSmaller() {
- }
+ public final static int smallerCount = 17;
- public MyTestClassSmaller(Byte b, Short s, Integer i, Long l, Float f,
- Double d, String st, HiveDecimal bd, Date date, MyTestInnerStruct is) {
- myByte = b;
- myShort = s;
- myInt = i;
- myLong = l;
- myFloat = f;
- myDouble = d;
- myString = st;
- myDecimal = bd;
- myDate = date;
- myStruct = is;
+ public int randomFill(Random r, ExtraTypeInfo extraTypeInfo) {
+ int randField = r.nextInt(smallerCount);
+ int field = 0;
+
+ myBool = (randField == field++) ? null : (r.nextInt(1) == 1);
+ myByte = (randField == field++) ? null : Byte.valueOf((byte) r.nextInt());
+ myShort = (randField == field++) ? null : Short.valueOf((short) r.nextInt());
+ myInt = (randField == field++) ? null : Integer.valueOf(r.nextInt());
+ myLong = (randField == field++) ? null : Long.valueOf(r.nextLong());
+ myFloat = (randField == field++) ? null : Float
+ .valueOf(r.nextFloat() * 10 - 5);
+ myDouble = (randField == field++) ? null : Double
+ .valueOf(r.nextDouble() * 10 - 5);
+ myString = (randField == field++) ? null : MyTestPrimitiveClass.getRandString(r);
+ myHiveChar = (randField == field++) ? null : MyTestPrimitiveClass.getRandHiveChar(r, extraTypeInfo);
+ myHiveVarchar = (randField == field++) ? null : MyTestPrimitiveClass.getRandHiveVarchar(r, extraTypeInfo);
+ myBinary = MyTestPrimitiveClass.getRandBinary(r, r.nextInt(1000));
+ myDecimal = (randField == field++) ? null : MyTestPrimitiveClass.getRandHiveDecimal(r, extraTypeInfo);
+ myDate = (randField == field++) ? null : MyTestPrimitiveClass.getRandDate(r);
+ myTimestamp = (randField == field++) ? null : MyTestPrimitiveClass.getRandTimestamp(r);
+ myIntervalYearMonth = (randField == field++) ? null : MyTestPrimitiveClass.getRandIntervalYearMonth(r);
+ myIntervalDayTime = (randField == field++) ? null : MyTestPrimitiveClass.getRandIntervalDayTime(r);
+
+ myStruct = (randField == field++) ? null : new MyTestInnerStruct(
+ r.nextInt(5) - 2, r.nextInt(5) - 2);
+ return field;
}
+
}
Added: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java?rev=1673337&view=auto
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java (added)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java Tue Apr 14 01:59:29 2015
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.lazybinary;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.hive.serde2.ByteStream.Output;
+import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.serde2.VerifyFast;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo;
+import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableDeserializeRead;
+import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinaryDeserializeRead;
+import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinarySerializeWrite;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.io.BytesWritable;
+
+public class TestLazyBinaryFast extends TestCase {
+
+ private void testLazyBinaryFast(MyTestPrimitiveClass[] myTestPrimitiveClasses, SerDe[] serdes, StructObjectInspector[] rowOIs,
+ PrimitiveTypeInfo[][] primitiveTypeInfosArray) throws Throwable {
+
+ LazyBinarySerializeWrite lazyBinarySerializeWrite = new LazyBinarySerializeWrite(MyTestPrimitiveClass.primitiveCount);
+
+ // Try to serialize
+ BytesWritable serializeWriteBytes[] = new BytesWritable[myTestPrimitiveClasses.length];
+ for (int i = 0; i < myTestPrimitiveClasses.length; i++) {
+ MyTestPrimitiveClass t = myTestPrimitiveClasses[i];
+ Output output = new Output();
+ lazyBinarySerializeWrite.set(output);
+
+ for (int index = 0; index < MyTestPrimitiveClass.primitiveCount; index++) {
+ Object object = t.getPrimitiveObject(index);
+ PrimitiveCategory primitiveCategory = t.getPrimitiveCategory(index);
+ VerifyFast.serializeWrite(lazyBinarySerializeWrite, primitiveCategory, object);
+ }
+
+ BytesWritable bytesWritable = new BytesWritable();
+ bytesWritable.set(output.getData(), 0, output.getLength());
+ serializeWriteBytes[i] = bytesWritable;
+ }
+
+ // Try to deserialize
+ for (int i = 0; i < myTestPrimitiveClasses.length; i++) {
+ MyTestPrimitiveClass t = myTestPrimitiveClasses[i];
+ PrimitiveTypeInfo[] primitiveTypeInfos = primitiveTypeInfosArray[i];
+ LazyBinaryDeserializeRead lazyBinaryDeserializeRead =
+ new LazyBinaryDeserializeRead(primitiveTypeInfos);
+
+ BytesWritable bytesWritable = serializeWriteBytes[i];
+ lazyBinaryDeserializeRead.set(bytesWritable.getBytes(), 0, bytesWritable.getLength());
+
+ for (int index = 0; index < MyTestPrimitiveClass.primitiveCount; index++) {
+ Object object = t.getPrimitiveObject(index);
+ PrimitiveCategory primitiveCategory = t.getPrimitiveCategory(index);
+ VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, primitiveTypeInfos[index], object);
+ }
+ lazyBinaryDeserializeRead.extraFieldsCheck();
+ TestCase.assertTrue(!lazyBinaryDeserializeRead.readBeyondConfiguredFieldsWarned());
+ TestCase.assertTrue(!lazyBinaryDeserializeRead.readBeyondBufferRangeWarned());
+ TestCase.assertTrue(!lazyBinaryDeserializeRead.bufferRangeHasExtraDataWarned());
+ }
+
+ // Try to deserialize using SerDe class our Writable row objects created by SerializeWrite.
+ for (int i = 0; i < myTestPrimitiveClasses.length; i++) {
+ BytesWritable bytesWritable = serializeWriteBytes[i];
+ LazyBinaryStruct lazyBinaryStruct = (LazyBinaryStruct) serdes[i].deserialize(bytesWritable);
+
+ MyTestPrimitiveClass t = myTestPrimitiveClasses[i];
+ PrimitiveTypeInfo[] primitiveTypeInfos = primitiveTypeInfosArray[i];
+
+ for (int index = 0; index < MyTestPrimitiveClass.primitiveCount; index++) {
+ PrimitiveTypeInfo primitiveTypeInfo = primitiveTypeInfos[index];
+ Object expected = t.getPrimitiveWritableObject(index, primitiveTypeInfo);
+ Object object = lazyBinaryStruct.getField(index);
+ if (expected == null || object == null) {
+ if (expected != null || object != null) {
+ fail("SerDe deserialized NULL column mismatch");
+ }
+ } else {
+ if (!object.equals(expected)) {
+ fail("SerDe deserialized value does not match");
+ }
+ }
+ }
+ }
+
+ // One Writable per row.
+ BytesWritable serdeBytes[] = new BytesWritable[myTestPrimitiveClasses.length];
+
+ // Serialize using the SerDe, then below deserialize using DeserializeRead.
+ Object[] row = new Object[MyTestPrimitiveClass.primitiveCount];
+ for (int i = 0; i < myTestPrimitiveClasses.length; i++) {
+ MyTestPrimitiveClass t = myTestPrimitiveClasses[i];
+ PrimitiveTypeInfo[] primitiveTypeInfos = primitiveTypeInfosArray[i];
+
+ // LazyBinary seems to work better with an row object array instead of a Java object...
+ for (int index = 0; index < MyTestPrimitiveClass.primitiveCount; index++) {
+ Object object = t.getPrimitiveWritableObject(index, primitiveTypeInfos[index]);
+ row[index] = object;
+ }
+
+ BytesWritable serialized = (BytesWritable) serdes[i].serialize(row, rowOIs[i]);
+ BytesWritable bytesWritable = new BytesWritable();
+ bytesWritable.set(serialized);
+ byte[] bytes1 = Arrays.copyOfRange(bytesWritable.getBytes(), 0, bytesWritable.getLength());
+
+ byte[] bytes2 = Arrays.copyOfRange(serializeWriteBytes[i].getBytes(), 0, serializeWriteBytes[i].getLength());
+ if (!Arrays.equals(bytes1, bytes2)) {
+ fail("SerializeWrite and SerDe serialization does not match");
+ }
+ serdeBytes[i] = bytesWritable;
+ }
+
+ // Try to deserialize using DeserializeRead our Writable row objects created by SerDe.
+ for (int i = 0; i < myTestPrimitiveClasses.length; i++) {
+ MyTestPrimitiveClass t = myTestPrimitiveClasses[i];
+ PrimitiveTypeInfo[] primitiveTypeInfos = primitiveTypeInfosArray[i];
+ LazyBinaryDeserializeRead lazyBinaryDeserializeRead =
+ new LazyBinaryDeserializeRead(primitiveTypeInfos);
+
+ BytesWritable bytesWritable = serdeBytes[i];
+ lazyBinaryDeserializeRead.set(bytesWritable.getBytes(), 0, bytesWritable.getLength());
+
+ for (int index = 0; index < MyTestPrimitiveClass.primitiveCount; index++) {
+ Object object = t.getPrimitiveObject(index);
+ VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, primitiveTypeInfos[index], object);
+ }
+ lazyBinaryDeserializeRead.extraFieldsCheck();
+ TestCase.assertTrue(!lazyBinaryDeserializeRead.readBeyondConfiguredFieldsWarned());
+ TestCase.assertTrue(!lazyBinaryDeserializeRead.readBeyondBufferRangeWarned());
+ TestCase.assertTrue(!lazyBinaryDeserializeRead.bufferRangeHasExtraDataWarned());
+ }
+ }
+
+ public void testLazyBinaryFast() throws Throwable {
+ try {
+
+ int num = 1000;
+ Random r = new Random(1234);
+ MyTestPrimitiveClass[] rows = new MyTestPrimitiveClass[num];
+ PrimitiveTypeInfo[][] primitiveTypeInfosArray = new PrimitiveTypeInfo[num][];
+ for (int i = 0; i < num; i++) {
+ int randField = r.nextInt(MyTestPrimitiveClass.primitiveCount);
+ MyTestPrimitiveClass t = new MyTestPrimitiveClass();
+ int field = 0;
+ ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
+ t.randomFill(r, randField, field, extraTypeInfo);
+ PrimitiveTypeInfo[] primitiveTypeInfos = MyTestPrimitiveClass.getPrimitiveTypeInfos(extraTypeInfo);
+ rows[i] = t;
+ primitiveTypeInfosArray[i] = primitiveTypeInfos;
+ }
+
+ // To get the specific type information for CHAR and VARCHAR, seems like we need an
+ // inspector and SerDe per row...
+ StructObjectInspector[] rowOIs = new StructObjectInspector[num];
+ SerDe[] serdes = new SerDe[num];
+ for (int i = 0; i < num; i++) {
+ MyTestPrimitiveClass t = rows[i];
+
+ StructObjectInspector rowOI = t.getRowInspector(primitiveTypeInfosArray[i]);
+
+ String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI);
+ String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI);
+
+ rowOIs[i] = rowOI;
+ serdes[i] = TestLazyBinarySerDe.getSerDe(fieldNames, fieldTypes);
+ }
+
+ testLazyBinaryFast(rows, serdes, rowOIs, primitiveTypeInfosArray);
+ } catch (Throwable e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+}
\ No newline at end of file
Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java?rev=1673337&r1=1673336&r2=1673337&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java Tue Apr 14 01:59:29 2015
@@ -36,7 +36,9 @@ import org.apache.hadoop.hive.serde2.Ser
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.serde2.binarysortable.MyTestClass;
import org.apache.hadoop.hive.serde2.binarysortable.MyTestInnerStruct;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass;
import org.apache.hadoop.hive.serde2.binarysortable.TestBinarySortableSerDe;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo;
import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
import org.apache.hadoop.hive.serde2.lazy.LazyBinary;
import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
@@ -91,7 +93,7 @@ public class TestLazyBinarySerDe extends
* @return the initialized LazyBinarySerDe
* @throws Throwable
*/
- private SerDe getSerDe(String fieldNames, String fieldTypes) throws Throwable {
+ protected static SerDe getSerDe(String fieldNames, String fieldTypes) throws Throwable {
Properties schema = new Properties();
schema.setProperty(serdeConstants.LIST_COLUMNS, fieldNames);
schema.setProperty(serdeConstants.LIST_COLUMN_TYPES, fieldTypes);
@@ -194,46 +196,20 @@ public class TestLazyBinarySerDe extends
int num = 100;
for (int itest = 0; itest < num; itest++) {
- int randField = r.nextInt(11);
- Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt());
- Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt());
- Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt());
- Long l = randField > 3 ? null : Long.valueOf(r.nextLong());
- Float f = randField > 4 ? null : Float.valueOf(r.nextFloat());
- Double d = randField > 5 ? null : Double.valueOf(r.nextDouble());
- String st = randField > 6 ? null : TestBinarySortableSerDe
- .getRandString(r);
- HiveDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandHiveDecimal(r);
- Date date = randField > 8 ? null : TestBinarySortableSerDe.getRandDate(r);
- MyTestInnerStruct is = randField > 9 ? null : new MyTestInnerStruct(r
- .nextInt(5) - 2, r.nextInt(5) - 2);
- List<Integer> li = randField > 10 ? null : TestBinarySortableSerDe
- .getRandIntegerArray(r);
- byte[] ba = TestBinarySortableSerDe.getRandBA(r, itest);
- Map<String, List<MyTestInnerStruct>> mp = new HashMap<String, List<MyTestInnerStruct>>();
- String key = TestBinarySortableSerDe.getRandString(r);
- List<MyTestInnerStruct> value = randField > 9 ? null
- : getRandStructArray(r);
- mp.put(key, value);
- String key1 = TestBinarySortableSerDe.getRandString(r);
- mp.put(key1, null);
- String key2 = TestBinarySortableSerDe.getRandString(r);
- List<MyTestInnerStruct> value2 = getRandStructArray(r);
- mp.put(key2, value2);
-
- MyTestClassBigger input = new MyTestClassBigger(b, s, n, l, f, d, st, bd, date, is,
- li, ba, mp);
- BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1);
+ MyTestClassBigger t = new MyTestClassBigger();
+ ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
+ t.randomFill(r, extraTypeInfo);
+ BytesWritable bw = (BytesWritable) serde1.serialize(t, rowOI1);
Object output = serde2.deserialize(bw);
- if (0 != compareDiffSizedStructs(input, rowOI1, output, serdeOI2)) {
+ if (0 != compareDiffSizedStructs(t, rowOI1, output, serdeOI2)) {
System.out.println("structs = "
- + SerDeUtils.getJSONString(input, rowOI1));
+ + SerDeUtils.getJSONString(t, rowOI1));
System.out.println("deserialized = "
+ SerDeUtils.getJSONString(output, serdeOI2));
System.out.println("serialized = "
+ TestBinarySortableSerDe.hexString(bw));
- assertEquals(input, output);
+ assertEquals(t, output);
}
}
}
@@ -263,34 +239,20 @@ public class TestLazyBinarySerDe extends
int num = 100;
for (int itest = 0; itest < num; itest++) {
- int randField = r.nextInt(12);
- Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt());
- Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt());
- Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt());
- Long l = randField > 3 ? null : Long.valueOf(r.nextLong());
- Float f = randField > 4 ? null : Float.valueOf(r.nextFloat());
- Double d = randField > 5 ? null : Double.valueOf(r.nextDouble());
- String st = randField > 6 ? null : TestBinarySortableSerDe
- .getRandString(r);
- HiveDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandHiveDecimal(r);
- Date date = randField > 8 ? null : TestBinarySortableSerDe.getRandDate(r);
- MyTestInnerStruct is = randField > 9 ? null : new MyTestInnerStruct(r
- .nextInt(5) - 2, r.nextInt(5) - 2);
- List<Integer> li = randField > 10 ? null : TestBinarySortableSerDe
- .getRandIntegerArray(r);
- byte[] ba = TestBinarySortableSerDe.getRandBA(r, itest);
- MyTestClass input = new MyTestClass(b, s, n, l, f, d, st, bd, date, is, li, ba);
- BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1);
+ MyTestClass t = new MyTestClass();
+ ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
+ t.randomFill(r, extraTypeInfo);
+ BytesWritable bw = (BytesWritable) serde1.serialize(t, rowOI1);
Object output = serde2.deserialize(bw);
- if (0 != compareDiffSizedStructs(input, rowOI1, output, serdeOI2)) {
+ if (0 != compareDiffSizedStructs(t, rowOI1, output, serdeOI2)) {
System.out.println("structs = "
- + SerDeUtils.getJSONString(input, rowOI1));
+ + SerDeUtils.getJSONString(t, rowOI1));
System.out.println("deserialized = "
+ SerDeUtils.getJSONString(output, serdeOI2));
System.out.println("serialized = "
+ TestBinarySortableSerDe.hexString(bw));
- assertEquals(input, output);
+ assertEquals(t, output);
}
}
}
@@ -320,34 +282,21 @@ public class TestLazyBinarySerDe extends
int num = 100;
for (int itest = 0; itest < num; itest++) {
- int randField = r.nextInt(12);
- Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt());
- Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt());
- Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt());
- Long l = randField > 3 ? null : Long.valueOf(r.nextLong());
- Float f = randField > 4 ? null : Float.valueOf(r.nextFloat());
- Double d = randField > 5 ? null : Double.valueOf(r.nextDouble());
- String st = randField > 6 ? null : TestBinarySortableSerDe
- .getRandString(r);
- HiveDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandHiveDecimal(r);
- Date date = randField > 8 ? null : TestBinarySortableSerDe.getRandDate(r);
- MyTestInnerStruct is = randField > 9 ? null : new MyTestInnerStruct(r
- .nextInt(5) - 2, r.nextInt(5) - 2);
- List<Integer> li = randField > 10 ? null : TestBinarySortableSerDe
- .getRandIntegerArray(r);
- byte[] ba = TestBinarySortableSerDe.getRandBA(r, itest);
- MyTestClass input = new MyTestClass(b, s, n, l, f, d, st, bd, date, is, li,ba);
- BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1);
+ MyTestClass t = new MyTestClass();
+ ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
+ t.randomFill(r, extraTypeInfo);
+
+ BytesWritable bw = (BytesWritable) serde1.serialize(t, rowOI1);
Object output = serde2.deserialize(bw);
- if (0 != compareDiffSizedStructs(input, rowOI1, output, serdeOI2)) {
+ if (0 != compareDiffSizedStructs(t, rowOI1, output, serdeOI2)) {
System.out.println("structs = "
- + SerDeUtils.getJSONString(input, rowOI1));
+ + SerDeUtils.getJSONString(t, rowOI1));
System.out.println("deserialized = "
+ SerDeUtils.getJSONString(output, serdeOI2));
System.out.println("serialized = "
+ TestBinarySortableSerDe.hexString(bw));
- assertEquals(input, output);
+ assertEquals(t, output);
}
}
}
@@ -377,33 +326,20 @@ public class TestLazyBinarySerDe extends
int num = 100;
for (int itest = 0; itest < num; itest++) {
- int randField = r.nextInt(9);
- Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt());
- Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt());
- Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt());
- Long l = randField > 3 ? null : Long.valueOf(r.nextLong());
- Float f = randField > 4 ? null : Float.valueOf(r.nextFloat());
- Double d = randField > 5 ? null : Double.valueOf(r.nextDouble());
- String st = randField > 6 ? null : TestBinarySortableSerDe
- .getRandString(r);
- HiveDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandHiveDecimal(r);
- Date date = randField > 7 ? null : TestBinarySortableSerDe.getRandDate(r);
- MyTestInnerStruct is = randField > 7 ? null : new MyTestInnerStruct(r
- .nextInt(5) - 2, r.nextInt(5) - 2);
-
- MyTestClassSmaller input = new MyTestClassSmaller(b, s, n, l, f, d, st, bd, date,
- is);
- BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1);
+ MyTestClassSmaller t = new MyTestClassSmaller();
+ ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
+ t.randomFill(r, extraTypeInfo);
+ BytesWritable bw = (BytesWritable) serde1.serialize(t, rowOI1);
Object output = serde2.deserialize(bw);
- if (0 != compareDiffSizedStructs(input, rowOI1, output, serdeOI2)) {
+ if (0 != compareDiffSizedStructs(t, rowOI1, output, serdeOI2)) {
System.out.println("structs = "
- + SerDeUtils.getJSONString(input, rowOI1));
+ + SerDeUtils.getJSONString(t, rowOI1));
System.out.println("deserialized = "
+ SerDeUtils.getJSONString(output, serdeOI2));
System.out.println("serialized = "
+ TestBinarySortableSerDe.hexString(bw));
- assertEquals(input, output);
+ assertEquals(t, output);
}
}
}
@@ -421,13 +357,13 @@ public class TestLazyBinarySerDe extends
StructObjectInspector soi1 = (StructObjectInspector) serdeOI;
List<? extends StructField> fields1 = soi1.getAllStructFieldRefs();
LazyBinaryMapObjectInspector lazympoi = (LazyBinaryMapObjectInspector) fields1
- .get(12).getFieldObjectInspector();
+ .get(MyTestClassBigger.mapPos).getFieldObjectInspector();
ObjectInspector lazympkeyoi = lazympoi.getMapKeyObjectInspector();
ObjectInspector lazympvalueoi = lazympoi.getMapValueObjectInspector();
StructObjectInspector soi2 = rowOI;
List<? extends StructField> fields2 = soi2.getAllStructFieldRefs();
- MapObjectInspector inputmpoi = (MapObjectInspector) fields2.get(12)
+ MapObjectInspector inputmpoi = (MapObjectInspector) fields2.get(MyTestClassBigger.mapPos)
.getFieldObjectInspector();
ObjectInspector inputmpkeyoi = inputmpoi.getMapKeyObjectInspector();
ObjectInspector inputmpvalueoi = inputmpoi.getMapValueObjectInspector();
@@ -439,18 +375,19 @@ public class TestLazyBinarySerDe extends
int randFields = r.nextInt(10);
for (int i = 0; i < randFields; i++) {
- String key = TestBinarySortableSerDe.getRandString(r);
+ String key = MyTestPrimitiveClass.getRandString(r);
int randField = r.nextInt(10);
List<MyTestInnerStruct> value = randField > 4 ? null
: getRandStructArray(r);
mp.put(key, value);
+
}
- MyTestClassBigger input = new MyTestClassBigger(null, null, null, null,
- null, null, null, null, null, null, null, null, mp);
- BytesWritable bw = (BytesWritable) serde.serialize(input, rowOI);
+ MyTestClassBigger t = new MyTestClassBigger();
+ t.myMap = mp;
+ BytesWritable bw = (BytesWritable) serde.serialize(t, rowOI);
Object output = serde.deserialize(bw);
- Object lazyobj = soi1.getStructFieldData(output, fields1.get(12));
+ Object lazyobj = soi1.getStructFieldData(output, fields1.get(MyTestClassBigger.mapPos));
Map<?, ?> outputmp = lazympoi.getMap(lazyobj);
if (outputmp.size() != mp.size()) {
@@ -497,23 +434,9 @@ public class TestLazyBinarySerDe extends
Random r = new Random(1234);
MyTestClass rows[] = new MyTestClass[num];
for (int i = 0; i < num; i++) {
- int randField = r.nextInt(12);
- Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt());
- Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt());
- Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt());
- Long l = randField > 3 ? null : Long.valueOf(r.nextLong());
- Float f = randField > 4 ? null : Float.valueOf(r.nextFloat());
- Double d = randField > 5 ? null : Double.valueOf(r.nextDouble());
- String st = randField > 6 ? null : TestBinarySortableSerDe
- .getRandString(r);
- HiveDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandHiveDecimal(r);
- Date date = randField > 8 ? null : TestBinarySortableSerDe.getRandDate(r);
- MyTestInnerStruct is = randField > 9 ? null : new MyTestInnerStruct(r
- .nextInt(5) - 2, r.nextInt(5) - 2);
- List<Integer> li = randField > 10 ? null : TestBinarySortableSerDe
- .getRandIntegerArray(r);
- byte[] ba = TestBinarySortableSerDe.getRandBA(r, i);
- MyTestClass t = new MyTestClass(b, s, n, l, f, d, st, bd, date, is, li, ba);
+ MyTestClass t = new MyTestClass();
+ ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
+ t.randomFill(r, extraTypeInfo);
rows[i] = t;
}