You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2015/04/14 03:59:31 UTC

svn commit: r1673337 [10/10] - in /hive/trunk: common/src/java/org/apache/hadoop/hive/common/type/ common/src/java/org/apache/hive/common/util/ itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql...

Added: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java?rev=1673337&view=auto
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java (added)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleFast.java Tue Apr 14 01:59:29 2015
@@ -0,0 +1,261 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.lazy;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.ByteStream.Output;
+import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.SerDeUtils;
+import org.apache.hadoop.hive.serde2.VerifyFast;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo;
+import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableDeserializeRead;
+import org.apache.hadoop.hive.serde2.lazy.fast.LazySimpleDeserializeRead;
+import org.apache.hadoop.hive.serde2.lazy.fast.LazySimpleSerializeWrite;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+
+public class TestLazySimpleFast extends TestCase {
+
+  private void testLazySimpleFast(MyTestPrimitiveClass[] myTestPrimitiveClasses, LazySimpleSerDe[] serdes,
+      StructObjectInspector[] rowOIs, byte separator, LazySerDeParameters[] serdeParams,
+      PrimitiveTypeInfo[][] primitiveTypeInfosArray) throws Throwable {
+
+
+    // Try to serialize
+    BytesWritable serializeWriteBytes[] = new BytesWritable[myTestPrimitiveClasses.length];
+    for (int i = 0; i < myTestPrimitiveClasses.length; i++) {
+      MyTestPrimitiveClass t = myTestPrimitiveClasses[i];
+      Output output = new Output();
+
+      LazySimpleSerializeWrite lazySimpleSerializeWrite = 
+          new LazySimpleSerializeWrite(MyTestPrimitiveClass.primitiveCount,
+              separator, serdeParams[i]);
+
+      lazySimpleSerializeWrite.set(output);
+
+      for (int index = 0; index < MyTestPrimitiveClass.primitiveCount; index++) {
+        Object object = t.getPrimitiveObject(index);
+        PrimitiveCategory primitiveCategory = t.getPrimitiveCategory(index);
+        VerifyFast.serializeWrite(lazySimpleSerializeWrite, primitiveCategory, object);
+      }
+
+      BytesWritable bytesWritable = new BytesWritable();
+      bytesWritable.set(output.getData(), 0, output.getLength());
+      serializeWriteBytes[i] = bytesWritable;
+    }
+
+    // Try to deserialize
+    for (int i = 0; i < myTestPrimitiveClasses.length; i++) {
+      MyTestPrimitiveClass t = myTestPrimitiveClasses[i];
+      PrimitiveTypeInfo[] primitiveTypeInfos = primitiveTypeInfosArray[i];
+      LazySimpleDeserializeRead lazySimpleDeserializeRead = 
+              new LazySimpleDeserializeRead(primitiveTypeInfos,
+                  separator, serdeParams[i]);
+
+      BytesWritable bytesWritable = serializeWriteBytes[i];
+      byte[] bytes = bytesWritable.getBytes();
+      int length = bytesWritable.getLength();
+      lazySimpleDeserializeRead.set(bytes, 0, length);
+
+      char[] chars = new char[length];
+      for (int c = 0; c < chars.length; c++) {
+        chars[c] = (char) (bytes[c] & 0xFF);
+      }
+
+      for (int index = 0; index < MyTestPrimitiveClass.primitiveCount; index++) {
+        Object object = t.getPrimitiveObject(index);
+        PrimitiveCategory primitiveCategory = t.getPrimitiveCategory(index);
+        VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, primitiveTypeInfos[index], object);
+      }
+      lazySimpleDeserializeRead.extraFieldsCheck();
+      TestCase.assertTrue(!lazySimpleDeserializeRead.readBeyondConfiguredFieldsWarned());
+      TestCase.assertTrue(!lazySimpleDeserializeRead.readBeyondBufferRangeWarned());
+      TestCase.assertTrue(!lazySimpleDeserializeRead.bufferRangeHasExtraDataWarned());
+    }
+
+    // Try to deserialize using SerDe class our Writable row objects created by SerializeWrite.
+    for (int i = 0; i < myTestPrimitiveClasses.length; i++) {
+      BytesWritable bytesWritable = serializeWriteBytes[i];
+      LazyStruct lazySimpleStruct = (LazyStruct) serdes[i].deserialize(bytesWritable);
+
+      MyTestPrimitiveClass t = myTestPrimitiveClasses[i];
+      PrimitiveTypeInfo[] primitiveTypeInfos = primitiveTypeInfosArray[i];
+
+      for (int index = 0; index < MyTestPrimitiveClass.primitiveCount; index++) {
+        PrimitiveTypeInfo primitiveTypeInfo = primitiveTypeInfos[index];
+        Object expected = t.getPrimitiveWritableObject(index, primitiveTypeInfo);
+        LazyPrimitive lazyPrimitive = (LazyPrimitive) lazySimpleStruct.getField(index);
+        Object object;
+        if (lazyPrimitive != null) {
+          object = lazyPrimitive.getWritableObject();
+        } else {
+          object = null;
+        }
+        if (expected == null || object == null) {
+          if (expected != null || object != null) {
+            fail("SerDe deserialized NULL column mismatch");
+          }
+        } else {
+          if (!object.equals(expected)) {
+            fail("SerDe deserialized value does not match");
+          }
+        }
+      }
+    }
+
+    // One Writable per row.
+    byte[][] serdeBytes = new byte[myTestPrimitiveClasses.length][];
+  
+    // Serialize using the SerDe, then below deserialize using DeserializeRead.
+    Object[] row = new Object[MyTestPrimitiveClass.primitiveCount];
+    for (int i = 0; i < myTestPrimitiveClasses.length; i++) {
+      MyTestPrimitiveClass t = myTestPrimitiveClasses[i];
+      PrimitiveTypeInfo[] primitiveTypeInfos = primitiveTypeInfosArray[i];
+
+      // LazySimple seems to work better with an row object array instead of a Java object...
+      for (int index = 0; index < MyTestPrimitiveClass.primitiveCount; index++) {
+        Object object = t.getPrimitiveWritableObject(index, primitiveTypeInfos[index]);
+        row[index] = object;
+      }
+
+      Text serialized = (Text) serdes[i].serialize(row, rowOIs[i]);
+      byte[] bytes1 = Arrays.copyOfRange(serialized.getBytes(), 0, serialized.getLength());
+
+      byte[] bytes2 = Arrays.copyOfRange(serializeWriteBytes[i].getBytes(), 0, serializeWriteBytes[i].getLength());
+      if (!Arrays.equals(bytes1, bytes2)) {
+        fail("SerializeWrite and SerDe serialization does not match");
+      }
+      serdeBytes[i] = serialized.copyBytes();
+    }
+
+    // Try to deserialize using DeserializeRead our Writable row objects created by SerDe.
+    for (int i = 0; i < myTestPrimitiveClasses.length; i++) {
+      MyTestPrimitiveClass t = myTestPrimitiveClasses[i];
+      PrimitiveTypeInfo[] primitiveTypeInfos = primitiveTypeInfosArray[i];
+      LazySimpleDeserializeRead lazySimpleDeserializeRead = 
+              new LazySimpleDeserializeRead(primitiveTypeInfos,
+                  separator, serdeParams[i]);
+
+      byte[] bytes = serdeBytes[i];
+      lazySimpleDeserializeRead.set(bytes, 0, bytes.length);
+
+      for (int index = 0; index < MyTestPrimitiveClass.primitiveCount; index++) {
+        Object object = t.getPrimitiveObject(index);
+        VerifyFast.verifyDeserializeRead(lazySimpleDeserializeRead, primitiveTypeInfos[index], object);
+      }
+      lazySimpleDeserializeRead.extraFieldsCheck();
+      TestCase.assertTrue(!lazySimpleDeserializeRead.readBeyondConfiguredFieldsWarned());
+      TestCase.assertTrue(!lazySimpleDeserializeRead.readBeyondBufferRangeWarned());
+      TestCase.assertTrue(!lazySimpleDeserializeRead.bufferRangeHasExtraDataWarned());
+    }
+  }
+
+  private Properties createProperties(String fieldNames, String fieldTypes) {
+    Properties tbl = new Properties();
+
+    // Set the configuration parameters
+    tbl.setProperty(serdeConstants.SERIALIZATION_FORMAT, "9");
+    
+    tbl.setProperty("columns", fieldNames);
+    tbl.setProperty("columns.types", fieldTypes);
+
+    tbl.setProperty(serdeConstants.SERIALIZATION_NULL_FORMAT, "NULL");
+
+    return tbl;
+  }
+
+  private LazySimpleSerDe getSerDe(String fieldNames, String fieldTypes) throws SerDeException {
+    // Create the SerDe
+    LazySimpleSerDe serDe = new LazySimpleSerDe();
+    Configuration conf = new Configuration();
+    Properties tbl = createProperties(fieldNames, fieldTypes);
+    SerDeUtils.initializeSerDe(serDe, conf, tbl, null);
+    return serDe;
+  }
+
+  private LazySerDeParameters getSerDeParams(String fieldNames, String fieldTypes) throws SerDeException {
+    Configuration conf = new Configuration();
+    Properties tbl = createProperties(fieldNames, fieldTypes);
+    return new LazySerDeParameters(conf, tbl, LazySimpleSerDe.class.getName());
+  }
+
+  public void testLazySimpleFast() throws Throwable {
+    try {
+
+      int num = 1000;
+      Random r = new Random(1234);
+      MyTestPrimitiveClass[] rows = new MyTestPrimitiveClass[num];
+      PrimitiveTypeInfo[][] primitiveTypeInfosArray = new PrimitiveTypeInfo[num][];
+      for (int i = 0; i < num; i++) {
+        int randField = r.nextInt(MyTestPrimitiveClass.primitiveCount);
+        MyTestPrimitiveClass t = new MyTestPrimitiveClass();
+        int field = 0;
+        ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
+        t.randomFill(r, randField, field, extraTypeInfo);
+        PrimitiveTypeInfo[] primitiveTypeInfos = MyTestPrimitiveClass.getPrimitiveTypeInfos(extraTypeInfo);
+        rows[i] = t;
+        primitiveTypeInfosArray[i] = primitiveTypeInfos;
+      }
+
+      // To get the specific type information for CHAR and VARCHAR, seems like we need an
+      // inspector and SerDe per row...
+      StructObjectInspector[] rowOIs = new StructObjectInspector[num];
+      LazySimpleSerDe[] serdes = new LazySimpleSerDe[num];
+      LazySerDeParameters[] serdeParams = new LazySerDeParameters[num];
+      for (int i = 0; i < num; i++) {
+        MyTestPrimitiveClass t = rows[i];
+
+        StructObjectInspector rowOI = t.getRowInspector(primitiveTypeInfosArray[i]);
+
+        String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI);
+        String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI);
+
+        rowOIs[i] = rowOI;
+        serdes[i] = getSerDe(fieldNames, fieldTypes);
+        serdeParams[i] = getSerDeParams(fieldNames, fieldTypes);
+      }
+
+      byte separator = (byte) '\t';
+      testLazySimpleFast(rows, serdes, rowOIs, separator, serdeParams, primitiveTypeInfosArray);
+    } catch (Throwable e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+}
\ No newline at end of file

Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassBigger.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassBigger.java?rev=1673337&r1=1673336&r2=1673337&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassBigger.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassBigger.java Tue Apr 14 01:59:29 2015
@@ -18,49 +18,119 @@
 package org.apache.hadoop.hive.serde2.lazybinary;
 
 import java.sql.Date;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 
+import org.apache.hadoop.hive.common.type.HiveChar;
 import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestClass;
 import org.apache.hadoop.hive.serde2.binarysortable.MyTestInnerStruct;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass;
+import org.apache.hadoop.hive.serde2.binarysortable.TestBinarySortableSerDe;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo;
 
 /**
  * MyTestClassBigger.
  *
  */
 public class MyTestClassBigger {
-    Byte myByte;
-    Short myShort;
-    Integer myInt;
-    Long myLong;
-    Float myFloat;
-    Double myDouble;
-    String myString;
-    HiveDecimal myDecimal;
-    Date myDate;
-    MyTestInnerStruct myStruct;
-    List<Integer> myList;
-    byte[] myBA;
+
+    // The primitives.
+    public Boolean myBool;
+    public Byte myByte;
+    public Short myShort;
+    public Integer myInt;
+    public Long myLong;
+    public Float myFloat;
+    public Double myDouble;
+    public String myString;
+    public HiveChar myHiveChar;
+    public HiveVarchar myHiveVarchar;
+    public byte[] myBinary;
+    public HiveDecimal myDecimal;
+    public Date myDate;
+    public Timestamp myTimestamp;
+    public HiveIntervalYearMonth myIntervalYearMonth;
+    public HiveIntervalDayTime myIntervalDayTime;
+
+
+    // Add more complex types.
+    public MyTestInnerStruct myStruct;
+    public  List<Integer> myList;
+
+    // Bigger addition.
     Map<String, List<MyTestInnerStruct>> myMap;
 
+    public final static int mapPos = 18;
+
     public MyTestClassBigger() {
     }
 
-    public MyTestClassBigger(Byte b, Short s, Integer i, Long l, Float f,
-			     Double d, String st, HiveDecimal bd, Date date, MyTestInnerStruct is, List<Integer> li,
-			     byte[] ba, Map<String, List<MyTestInnerStruct>> mp) {
-	myByte = b;
-	myShort = s;
-	myInt = i;
-	myLong = l;
-	myFloat = f;
-	myDouble = d;
-	myString = st;
-	myDecimal = bd;
-	myDate = date;
-	myStruct = is;
-	myList = li;
-	myBA = ba;
-	myMap = mp;
+    public final static int biggerCount = 19;
+
+    public int randomFill(Random r, ExtraTypeInfo extraTypeInfo) {
+      int randField = r.nextInt(biggerCount);
+      int field = 0;
+      myBool = (randField == field++) ? null : (r.nextInt(1) == 1);
+      myByte = (randField == field++) ? null : Byte.valueOf((byte) r.nextInt());
+      myShort = (randField == field++) ? null : Short.valueOf((short) r.nextInt());
+      myInt = (randField == field++) ? null : Integer.valueOf(r.nextInt());
+      myLong = (randField == field++) ? null : Long.valueOf(r.nextLong());
+      myFloat = (randField == field++) ? null : Float
+          .valueOf(r.nextFloat() * 10 - 5);
+      myDouble = (randField == field++) ? null : Double
+          .valueOf(r.nextDouble() * 10 - 5);
+      myString = (randField == field++) ? null : MyTestPrimitiveClass.getRandString(r);
+      myHiveChar = (randField == field++) ? null : MyTestPrimitiveClass.getRandHiveChar(r, extraTypeInfo);
+      myHiveVarchar = (randField == field++) ? null : MyTestPrimitiveClass.getRandHiveVarchar(r, extraTypeInfo);
+      myBinary = MyTestPrimitiveClass.getRandBinary(r, r.nextInt(1000));
+      myDecimal = (randField == field++) ? null : MyTestPrimitiveClass.getRandHiveDecimal(r, extraTypeInfo);
+      myDate = (randField == field++) ? null : MyTestPrimitiveClass.getRandDate(r);
+      myTimestamp = (randField == field++) ? null : MyTestPrimitiveClass.getRandTimestamp(r);
+      myIntervalYearMonth = (randField == field++) ? null : MyTestPrimitiveClass.getRandIntervalYearMonth(r);
+      myIntervalDayTime = (randField == field++) ? null : MyTestPrimitiveClass.getRandIntervalDayTime(r);
+
+      myStruct = (randField == field++) ? null : new MyTestInnerStruct(
+          r.nextInt(5) - 2, r.nextInt(5) - 2);
+      myList = (randField == field++) ? null : MyTestClass.getRandIntegerArray(r);
+
+      Map<String, List<MyTestInnerStruct>> mp = new HashMap<String, List<MyTestInnerStruct>>();
+      String key = MyTestPrimitiveClass.getRandString(r);
+      List<MyTestInnerStruct> value = randField > 9 ? null
+          : getRandStructArray(r);
+      mp.put(key, value);
+      String key1 = MyTestPrimitiveClass.getRandString(r);
+      mp.put(key1, null);
+      String key2 = MyTestPrimitiveClass.getRandString(r);
+      List<MyTestInnerStruct> value2 = getRandStructArray(r);
+      mp.put(key2, value2);
+      myMap = mp;
+      return field;
     }
+
+    /**
+     * Generate a random struct array.
+     *
+     * @param r
+     *          random number generator
+     * @return an struct array
+     */
+    static List<MyTestInnerStruct> getRandStructArray(Random r) {
+      int length = r.nextInt(10);
+      ArrayList<MyTestInnerStruct> result = new ArrayList<MyTestInnerStruct>(
+          length);
+      for (int i = 0; i < length; i++) {
+        MyTestInnerStruct ti = new MyTestInnerStruct(r.nextInt(), r.nextInt());
+        result.add(ti);
+      }
+      return result;
+    }
+
 }

Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassSmaller.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassSmaller.java?rev=1673337&r1=1673336&r2=1673337&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassSmaller.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/MyTestClassSmaller.java Tue Apr 14 01:59:29 2015
@@ -18,36 +18,68 @@
 package org.apache.hadoop.hive.serde2.lazybinary;
 
 import java.sql.Date;
+import java.sql.Timestamp;
+import java.util.Random;
 
+import org.apache.hadoop.hive.common.type.HiveChar;
 import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestClass;
 import org.apache.hadoop.hive.serde2.binarysortable.MyTestInnerStruct;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo;
 
 public class MyTestClassSmaller {
-    Byte myByte;
-    Short myShort;
-    Integer myInt;
-    Long myLong;
-    Float myFloat;
-    Double myDouble;
-    String myString;
-    HiveDecimal myDecimal;
-    Date myDate;
+
+    public Boolean myBool;
+    public Byte myByte;
+    public Short myShort;
+    public Integer myInt;
+    public Long myLong;
+    public Float myFloat;
+    public Double myDouble;
+    public String myString;
+    public HiveChar myHiveChar;
+    public HiveVarchar myHiveVarchar;
+    public byte[] myBinary;
+    public HiveDecimal myDecimal;
+    public Date myDate;
+    public Timestamp myTimestamp;
+    public HiveIntervalYearMonth myIntervalYearMonth;
+    public HiveIntervalDayTime myIntervalDayTime;
+
     MyTestInnerStruct myStruct;
 
-    public MyTestClassSmaller() {
-    }
+    public final static int smallerCount = 17;
 
-    public MyTestClassSmaller(Byte b, Short s, Integer i, Long l, Float f,
-			      Double d, String st, HiveDecimal bd, Date date, MyTestInnerStruct is) {
-	myByte = b;
-	myShort = s;
-	myInt = i;
-	myLong = l;
-	myFloat = f;
-	myDouble = d;
-	myString = st;
-	myDecimal = bd;
-	myDate = date;
-	myStruct = is;
+    public int randomFill(Random r, ExtraTypeInfo extraTypeInfo) {
+      int randField = r.nextInt(smallerCount);
+      int field = 0;
+
+      myBool = (randField == field++) ? null : (r.nextInt(1) == 1);
+      myByte = (randField == field++) ? null : Byte.valueOf((byte) r.nextInt());
+      myShort = (randField == field++) ? null : Short.valueOf((short) r.nextInt());
+      myInt = (randField == field++) ? null : Integer.valueOf(r.nextInt());
+      myLong = (randField == field++) ? null : Long.valueOf(r.nextLong());
+      myFloat = (randField == field++) ? null : Float
+          .valueOf(r.nextFloat() * 10 - 5);
+      myDouble = (randField == field++) ? null : Double
+          .valueOf(r.nextDouble() * 10 - 5);
+      myString = (randField == field++) ? null : MyTestPrimitiveClass.getRandString(r);
+      myHiveChar = (randField == field++) ? null : MyTestPrimitiveClass.getRandHiveChar(r, extraTypeInfo);
+      myHiveVarchar = (randField == field++) ? null : MyTestPrimitiveClass.getRandHiveVarchar(r, extraTypeInfo);
+      myBinary = MyTestPrimitiveClass.getRandBinary(r, r.nextInt(1000));
+      myDecimal = (randField == field++) ? null : MyTestPrimitiveClass.getRandHiveDecimal(r, extraTypeInfo);
+      myDate = (randField == field++) ? null : MyTestPrimitiveClass.getRandDate(r);
+      myTimestamp = (randField == field++) ? null : MyTestPrimitiveClass.getRandTimestamp(r);
+      myIntervalYearMonth = (randField == field++) ? null : MyTestPrimitiveClass.getRandIntervalYearMonth(r);
+      myIntervalDayTime = (randField == field++) ? null : MyTestPrimitiveClass.getRandIntervalDayTime(r);
+
+      myStruct = (randField == field++) ? null : new MyTestInnerStruct(
+          r.nextInt(5) - 2, r.nextInt(5) - 2);
+      return field;
     }
+
 }

Added: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java?rev=1673337&view=auto
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java (added)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinaryFast.java Tue Apr 14 01:59:29 2015
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.lazybinary;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.hive.serde2.ByteStream.Output;
+import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.serde2.VerifyFast;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo;
+import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableDeserializeRead;
+import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinaryDeserializeRead;
+import org.apache.hadoop.hive.serde2.lazybinary.fast.LazyBinarySerializeWrite;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.io.BytesWritable;
+
+public class TestLazyBinaryFast extends TestCase {
+
+  private void testLazyBinaryFast(MyTestPrimitiveClass[] myTestPrimitiveClasses, SerDe[] serdes, StructObjectInspector[] rowOIs,
+      PrimitiveTypeInfo[][] primitiveTypeInfosArray) throws Throwable {
+
+    LazyBinarySerializeWrite lazyBinarySerializeWrite = new LazyBinarySerializeWrite(MyTestPrimitiveClass.primitiveCount);
+
+    // Try to serialize
+    BytesWritable serializeWriteBytes[] = new BytesWritable[myTestPrimitiveClasses.length];
+    for (int i = 0; i < myTestPrimitiveClasses.length; i++) {
+      MyTestPrimitiveClass t = myTestPrimitiveClasses[i];
+      Output output = new Output();
+      lazyBinarySerializeWrite.set(output);
+
+      for (int index = 0; index < MyTestPrimitiveClass.primitiveCount; index++) {
+        Object object = t.getPrimitiveObject(index);
+        PrimitiveCategory primitiveCategory = t.getPrimitiveCategory(index);
+        VerifyFast.serializeWrite(lazyBinarySerializeWrite, primitiveCategory, object);
+      }
+
+      BytesWritable bytesWritable = new BytesWritable();
+      bytesWritable.set(output.getData(), 0, output.getLength());
+      serializeWriteBytes[i] = bytesWritable;
+    }
+
+    // Try to deserialize
+    for (int i = 0; i < myTestPrimitiveClasses.length; i++) {
+      MyTestPrimitiveClass t = myTestPrimitiveClasses[i];
+      PrimitiveTypeInfo[] primitiveTypeInfos = primitiveTypeInfosArray[i];
+      LazyBinaryDeserializeRead lazyBinaryDeserializeRead = 
+              new LazyBinaryDeserializeRead(primitiveTypeInfos);
+
+      BytesWritable bytesWritable = serializeWriteBytes[i];
+      lazyBinaryDeserializeRead.set(bytesWritable.getBytes(), 0, bytesWritable.getLength());
+
+      for (int index = 0; index < MyTestPrimitiveClass.primitiveCount; index++) {
+        Object object = t.getPrimitiveObject(index);
+        PrimitiveCategory primitiveCategory = t.getPrimitiveCategory(index);
+        VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, primitiveTypeInfos[index], object);
+      }
+      lazyBinaryDeserializeRead.extraFieldsCheck();
+      TestCase.assertTrue(!lazyBinaryDeserializeRead.readBeyondConfiguredFieldsWarned());
+      TestCase.assertTrue(!lazyBinaryDeserializeRead.readBeyondBufferRangeWarned());
+      TestCase.assertTrue(!lazyBinaryDeserializeRead.bufferRangeHasExtraDataWarned());
+    }
+
+    // Try to deserialize using SerDe class our Writable row objects created by SerializeWrite.
+    for (int i = 0; i < myTestPrimitiveClasses.length; i++) {
+      BytesWritable bytesWritable = serializeWriteBytes[i];
+      LazyBinaryStruct lazyBinaryStruct = (LazyBinaryStruct) serdes[i].deserialize(bytesWritable);
+
+      MyTestPrimitiveClass t = myTestPrimitiveClasses[i];
+      PrimitiveTypeInfo[] primitiveTypeInfos = primitiveTypeInfosArray[i];
+
+      for (int index = 0; index < MyTestPrimitiveClass.primitiveCount; index++) {
+        PrimitiveTypeInfo primitiveTypeInfo = primitiveTypeInfos[index];
+        Object expected = t.getPrimitiveWritableObject(index, primitiveTypeInfo);
+        Object object = lazyBinaryStruct.getField(index);
+        if (expected == null || object == null) {
+          if (expected != null || object != null) {
+            fail("SerDe deserialized NULL column mismatch");
+          }
+        } else {
+          if (!object.equals(expected)) {
+            fail("SerDe deserialized value does not match");
+          }
+        }
+      }
+    }
+
+    // One Writable per row.
+    BytesWritable serdeBytes[] = new BytesWritable[myTestPrimitiveClasses.length];
+  
+    // Serialize using the SerDe, then below deserialize using DeserializeRead.
+    Object[] row = new Object[MyTestPrimitiveClass.primitiveCount];
+    for (int i = 0; i < myTestPrimitiveClasses.length; i++) {
+      MyTestPrimitiveClass t = myTestPrimitiveClasses[i];
+      PrimitiveTypeInfo[] primitiveTypeInfos = primitiveTypeInfosArray[i];
+
+      // LazyBinary seems to work better with an row object array instead of a Java object...
+      for (int index = 0; index < MyTestPrimitiveClass.primitiveCount; index++) {
+        Object object = t.getPrimitiveWritableObject(index, primitiveTypeInfos[index]);
+        row[index] = object;
+      }
+
+      BytesWritable serialized = (BytesWritable) serdes[i].serialize(row, rowOIs[i]);
+      BytesWritable bytesWritable = new BytesWritable();
+      bytesWritable.set(serialized);
+      byte[] bytes1 = Arrays.copyOfRange(bytesWritable.getBytes(), 0, bytesWritable.getLength());
+
+      byte[] bytes2 = Arrays.copyOfRange(serializeWriteBytes[i].getBytes(), 0, serializeWriteBytes[i].getLength());
+      if (!Arrays.equals(bytes1, bytes2)) {
+        fail("SerializeWrite and SerDe serialization does not match");
+      }
+      serdeBytes[i] = bytesWritable;
+    }
+
+    // Try to deserialize using DeserializeRead our Writable row objects created by SerDe.
+    for (int i = 0; i < myTestPrimitiveClasses.length; i++) {
+      MyTestPrimitiveClass t = myTestPrimitiveClasses[i];
+      PrimitiveTypeInfo[] primitiveTypeInfos = primitiveTypeInfosArray[i];
+      LazyBinaryDeserializeRead lazyBinaryDeserializeRead = 
+              new LazyBinaryDeserializeRead(primitiveTypeInfos);
+
+      BytesWritable bytesWritable = serdeBytes[i];
+      lazyBinaryDeserializeRead.set(bytesWritable.getBytes(), 0, bytesWritable.getLength());
+
+      for (int index = 0; index < MyTestPrimitiveClass.primitiveCount; index++) {
+        Object object = t.getPrimitiveObject(index);
+        VerifyFast.verifyDeserializeRead(lazyBinaryDeserializeRead, primitiveTypeInfos[index], object);
+      }
+      lazyBinaryDeserializeRead.extraFieldsCheck();
+      TestCase.assertTrue(!lazyBinaryDeserializeRead.readBeyondConfiguredFieldsWarned());
+      TestCase.assertTrue(!lazyBinaryDeserializeRead.readBeyondBufferRangeWarned());
+      TestCase.assertTrue(!lazyBinaryDeserializeRead.bufferRangeHasExtraDataWarned());
+    }
+  }
+
+  public void testLazyBinaryFast() throws Throwable {
+    try {
+
+      int num = 1000;
+      Random r = new Random(1234);
+      MyTestPrimitiveClass[] rows = new MyTestPrimitiveClass[num];
+      PrimitiveTypeInfo[][] primitiveTypeInfosArray = new PrimitiveTypeInfo[num][];
+      for (int i = 0; i < num; i++) {
+        int randField = r.nextInt(MyTestPrimitiveClass.primitiveCount);
+        MyTestPrimitiveClass t = new MyTestPrimitiveClass();
+        int field = 0;
+        ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
+        t.randomFill(r, randField, field, extraTypeInfo);
+        PrimitiveTypeInfo[] primitiveTypeInfos = MyTestPrimitiveClass.getPrimitiveTypeInfos(extraTypeInfo);
+        rows[i] = t;
+        primitiveTypeInfosArray[i] = primitiveTypeInfos;
+      }
+
+      // To get the specific type information for CHAR and VARCHAR, seems like we need an
+      // inspector and SerDe per row...
+      StructObjectInspector[] rowOIs = new StructObjectInspector[num];
+      SerDe[] serdes = new SerDe[num];
+      for (int i = 0; i < num; i++) {
+        MyTestPrimitiveClass t = rows[i];
+
+        StructObjectInspector rowOI = t.getRowInspector(primitiveTypeInfosArray[i]);
+
+        String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI);
+        String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI);
+
+        rowOIs[i] = rowOI;
+        serdes[i] = TestLazyBinarySerDe.getSerDe(fieldNames, fieldTypes);
+      }
+
+      testLazyBinaryFast(rows, serdes, rowOIs, primitiveTypeInfosArray);
+    } catch (Throwable e) {
+      e.printStackTrace();
+      throw e;
+    }
+  }
+}
\ No newline at end of file

Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java?rev=1673337&r1=1673336&r2=1673337&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java Tue Apr 14 01:59:29 2015
@@ -36,7 +36,9 @@ import org.apache.hadoop.hive.serde2.Ser
 import org.apache.hadoop.hive.serde2.SerDeUtils;
 import org.apache.hadoop.hive.serde2.binarysortable.MyTestClass;
 import org.apache.hadoop.hive.serde2.binarysortable.MyTestInnerStruct;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass;
 import org.apache.hadoop.hive.serde2.binarysortable.TestBinarySortableSerDe;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo;
 import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
 import org.apache.hadoop.hive.serde2.lazy.LazyBinary;
 import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
@@ -91,7 +93,7 @@ public class TestLazyBinarySerDe extends
    * @return the initialized LazyBinarySerDe
    * @throws Throwable
    */
-  private SerDe getSerDe(String fieldNames, String fieldTypes) throws Throwable {
+  protected static SerDe getSerDe(String fieldNames, String fieldTypes) throws Throwable {
     Properties schema = new Properties();
     schema.setProperty(serdeConstants.LIST_COLUMNS, fieldNames);
     schema.setProperty(serdeConstants.LIST_COLUMN_TYPES, fieldTypes);
@@ -194,46 +196,20 @@ public class TestLazyBinarySerDe extends
 
     int num = 100;
     for (int itest = 0; itest < num; itest++) {
-      int randField = r.nextInt(11);
-      Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt());
-      Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt());
-      Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt());
-      Long l = randField > 3 ? null : Long.valueOf(r.nextLong());
-      Float f = randField > 4 ? null : Float.valueOf(r.nextFloat());
-      Double d = randField > 5 ? null : Double.valueOf(r.nextDouble());
-      String st = randField > 6 ? null : TestBinarySortableSerDe
-          .getRandString(r);
-      HiveDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandHiveDecimal(r);
-      Date date = randField > 8 ? null : TestBinarySortableSerDe.getRandDate(r);
-      MyTestInnerStruct is = randField > 9 ? null : new MyTestInnerStruct(r
-          .nextInt(5) - 2, r.nextInt(5) - 2);
-      List<Integer> li = randField > 10 ? null : TestBinarySortableSerDe
-          .getRandIntegerArray(r);
-      byte[] ba  = TestBinarySortableSerDe.getRandBA(r, itest);
-      Map<String, List<MyTestInnerStruct>> mp = new HashMap<String, List<MyTestInnerStruct>>();
-      String key = TestBinarySortableSerDe.getRandString(r);
-      List<MyTestInnerStruct> value = randField > 9 ? null
-          : getRandStructArray(r);
-      mp.put(key, value);
-      String key1 = TestBinarySortableSerDe.getRandString(r);
-      mp.put(key1, null);
-      String key2 = TestBinarySortableSerDe.getRandString(r);
-      List<MyTestInnerStruct> value2 = getRandStructArray(r);
-      mp.put(key2, value2);
-
-      MyTestClassBigger input = new MyTestClassBigger(b, s, n, l, f, d, st, bd, date, is,
-          li, ba, mp);
-      BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1);
+      MyTestClassBigger t = new MyTestClassBigger();
+      ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
+      t.randomFill(r, extraTypeInfo);
+      BytesWritable bw = (BytesWritable) serde1.serialize(t, rowOI1);
       Object output = serde2.deserialize(bw);
 
-      if (0 != compareDiffSizedStructs(input, rowOI1, output, serdeOI2)) {
+      if (0 != compareDiffSizedStructs(t, rowOI1, output, serdeOI2)) {
         System.out.println("structs      = "
-            + SerDeUtils.getJSONString(input, rowOI1));
+            + SerDeUtils.getJSONString(t, rowOI1));
         System.out.println("deserialized = "
             + SerDeUtils.getJSONString(output, serdeOI2));
         System.out.println("serialized   = "
             + TestBinarySortableSerDe.hexString(bw));
-        assertEquals(input, output);
+        assertEquals(t, output);
       }
     }
   }
@@ -263,34 +239,20 @@ public class TestLazyBinarySerDe extends
 
     int num = 100;
     for (int itest = 0; itest < num; itest++) {
-      int randField = r.nextInt(12);
-      Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt());
-      Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt());
-      Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt());
-      Long l = randField > 3 ? null : Long.valueOf(r.nextLong());
-      Float f = randField > 4 ? null : Float.valueOf(r.nextFloat());
-      Double d = randField > 5 ? null : Double.valueOf(r.nextDouble());
-      String st = randField > 6 ? null : TestBinarySortableSerDe
-          .getRandString(r);
-      HiveDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandHiveDecimal(r);
-      Date date = randField > 8 ? null : TestBinarySortableSerDe.getRandDate(r);
-      MyTestInnerStruct is = randField > 9 ? null : new MyTestInnerStruct(r
-          .nextInt(5) - 2, r.nextInt(5) - 2);
-      List<Integer> li = randField > 10 ? null : TestBinarySortableSerDe
-          .getRandIntegerArray(r);
-      byte[] ba = TestBinarySortableSerDe.getRandBA(r, itest);
-      MyTestClass input = new MyTestClass(b, s, n, l, f, d, st, bd, date, is, li, ba);
-      BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1);
+      MyTestClass t = new MyTestClass();
+      ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
+      t.randomFill(r, extraTypeInfo);
+      BytesWritable bw = (BytesWritable) serde1.serialize(t, rowOI1);
       Object output = serde2.deserialize(bw);
 
-      if (0 != compareDiffSizedStructs(input, rowOI1, output, serdeOI2)) {
+      if (0 != compareDiffSizedStructs(t, rowOI1, output, serdeOI2)) {
         System.out.println("structs      = "
-            + SerDeUtils.getJSONString(input, rowOI1));
+            + SerDeUtils.getJSONString(t, rowOI1));
         System.out.println("deserialized = "
             + SerDeUtils.getJSONString(output, serdeOI2));
         System.out.println("serialized   = "
             + TestBinarySortableSerDe.hexString(bw));
-        assertEquals(input, output);
+        assertEquals(t, output);
       }
     }
   }
@@ -320,34 +282,21 @@ public class TestLazyBinarySerDe extends
 
     int num = 100;
     for (int itest = 0; itest < num; itest++) {
-      int randField = r.nextInt(12);
-      Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt());
-      Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt());
-      Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt());
-      Long l = randField > 3 ? null : Long.valueOf(r.nextLong());
-      Float f = randField > 4 ? null : Float.valueOf(r.nextFloat());
-      Double d = randField > 5 ? null : Double.valueOf(r.nextDouble());
-      String st = randField > 6 ? null : TestBinarySortableSerDe
-          .getRandString(r);
-      HiveDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandHiveDecimal(r);
-      Date date = randField > 8 ? null : TestBinarySortableSerDe.getRandDate(r);
-      MyTestInnerStruct is = randField > 9 ? null : new MyTestInnerStruct(r
-          .nextInt(5) - 2, r.nextInt(5) - 2);
-      List<Integer> li = randField > 10 ? null : TestBinarySortableSerDe
-          .getRandIntegerArray(r);
-      byte[] ba = TestBinarySortableSerDe.getRandBA(r, itest);
-      MyTestClass input = new MyTestClass(b, s, n, l, f, d, st, bd, date, is, li,ba);
-      BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1);
+      MyTestClass t = new MyTestClass();
+      ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
+      t.randomFill(r, extraTypeInfo);
+
+      BytesWritable bw = (BytesWritable) serde1.serialize(t, rowOI1);
       Object output = serde2.deserialize(bw);
 
-      if (0 != compareDiffSizedStructs(input, rowOI1, output, serdeOI2)) {
+      if (0 != compareDiffSizedStructs(t, rowOI1, output, serdeOI2)) {
         System.out.println("structs      = "
-            + SerDeUtils.getJSONString(input, rowOI1));
+            + SerDeUtils.getJSONString(t, rowOI1));
         System.out.println("deserialized = "
             + SerDeUtils.getJSONString(output, serdeOI2));
         System.out.println("serialized   = "
             + TestBinarySortableSerDe.hexString(bw));
-        assertEquals(input, output);
+        assertEquals(t, output);
       }
     }
   }
@@ -377,33 +326,20 @@ public class TestLazyBinarySerDe extends
 
     int num = 100;
     for (int itest = 0; itest < num; itest++) {
-      int randField = r.nextInt(9);
-      Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt());
-      Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt());
-      Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt());
-      Long l = randField > 3 ? null : Long.valueOf(r.nextLong());
-      Float f = randField > 4 ? null : Float.valueOf(r.nextFloat());
-      Double d = randField > 5 ? null : Double.valueOf(r.nextDouble());
-      String st = randField > 6 ? null : TestBinarySortableSerDe
-          .getRandString(r);
-      HiveDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandHiveDecimal(r);
-      Date date = randField > 7 ? null : TestBinarySortableSerDe.getRandDate(r);
-      MyTestInnerStruct is = randField > 7 ? null : new MyTestInnerStruct(r
-          .nextInt(5) - 2, r.nextInt(5) - 2);
-
-      MyTestClassSmaller input = new MyTestClassSmaller(b, s, n, l, f, d, st, bd, date,
-          is);
-      BytesWritable bw = (BytesWritable) serde1.serialize(input, rowOI1);
+      MyTestClassSmaller t = new MyTestClassSmaller();
+      ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
+      t.randomFill(r, extraTypeInfo);
+      BytesWritable bw = (BytesWritable) serde1.serialize(t, rowOI1);
       Object output = serde2.deserialize(bw);
 
-      if (0 != compareDiffSizedStructs(input, rowOI1, output, serdeOI2)) {
+      if (0 != compareDiffSizedStructs(t, rowOI1, output, serdeOI2)) {
         System.out.println("structs      = "
-            + SerDeUtils.getJSONString(input, rowOI1));
+            + SerDeUtils.getJSONString(t, rowOI1));
         System.out.println("deserialized = "
             + SerDeUtils.getJSONString(output, serdeOI2));
         System.out.println("serialized   = "
             + TestBinarySortableSerDe.hexString(bw));
-        assertEquals(input, output);
+        assertEquals(t, output);
       }
     }
   }
@@ -421,13 +357,13 @@ public class TestLazyBinarySerDe extends
     StructObjectInspector soi1 = (StructObjectInspector) serdeOI;
     List<? extends StructField> fields1 = soi1.getAllStructFieldRefs();
     LazyBinaryMapObjectInspector lazympoi = (LazyBinaryMapObjectInspector) fields1
-        .get(12).getFieldObjectInspector();
+        .get(MyTestClassBigger.mapPos).getFieldObjectInspector();
     ObjectInspector lazympkeyoi = lazympoi.getMapKeyObjectInspector();
     ObjectInspector lazympvalueoi = lazympoi.getMapValueObjectInspector();
 
     StructObjectInspector soi2 = rowOI;
     List<? extends StructField> fields2 = soi2.getAllStructFieldRefs();
-    MapObjectInspector inputmpoi = (MapObjectInspector) fields2.get(12)
+    MapObjectInspector inputmpoi = (MapObjectInspector) fields2.get(MyTestClassBigger.mapPos)
         .getFieldObjectInspector();
     ObjectInspector inputmpkeyoi = inputmpoi.getMapKeyObjectInspector();
     ObjectInspector inputmpvalueoi = inputmpoi.getMapValueObjectInspector();
@@ -439,18 +375,19 @@ public class TestLazyBinarySerDe extends
 
       int randFields = r.nextInt(10);
       for (int i = 0; i < randFields; i++) {
-        String key = TestBinarySortableSerDe.getRandString(r);
+        String key = MyTestPrimitiveClass.getRandString(r);
         int randField = r.nextInt(10);
         List<MyTestInnerStruct> value = randField > 4 ? null
             : getRandStructArray(r);
         mp.put(key, value);
+
       }
 
-      MyTestClassBigger input = new MyTestClassBigger(null, null, null, null,
-						      null, null, null, null, null, null, null, null, mp);
-      BytesWritable bw = (BytesWritable) serde.serialize(input, rowOI);
+      MyTestClassBigger t = new MyTestClassBigger();
+      t.myMap = mp;
+      BytesWritable bw = (BytesWritable) serde.serialize(t, rowOI);
       Object output = serde.deserialize(bw);
-      Object lazyobj = soi1.getStructFieldData(output, fields1.get(12));
+      Object lazyobj = soi1.getStructFieldData(output, fields1.get(MyTestClassBigger.mapPos));
       Map<?, ?> outputmp = lazympoi.getMap(lazyobj);
 
       if (outputmp.size() != mp.size()) {
@@ -497,23 +434,9 @@ public class TestLazyBinarySerDe extends
       Random r = new Random(1234);
       MyTestClass rows[] = new MyTestClass[num];
       for (int i = 0; i < num; i++) {
-        int randField = r.nextInt(12);
-        Byte b = randField > 0 ? null : Byte.valueOf((byte) r.nextInt());
-        Short s = randField > 1 ? null : Short.valueOf((short) r.nextInt());
-        Integer n = randField > 2 ? null : Integer.valueOf(r.nextInt());
-        Long l = randField > 3 ? null : Long.valueOf(r.nextLong());
-        Float f = randField > 4 ? null : Float.valueOf(r.nextFloat());
-        Double d = randField > 5 ? null : Double.valueOf(r.nextDouble());
-        String st = randField > 6 ? null : TestBinarySortableSerDe
-            .getRandString(r);
-        HiveDecimal bd = randField > 7 ? null : TestBinarySortableSerDe.getRandHiveDecimal(r);
-        Date date = randField > 8 ? null : TestBinarySortableSerDe.getRandDate(r);
-        MyTestInnerStruct is = randField > 9 ? null : new MyTestInnerStruct(r
-            .nextInt(5) - 2, r.nextInt(5) - 2);
-        List<Integer> li = randField > 10 ? null : TestBinarySortableSerDe
-            .getRandIntegerArray(r);
-        byte[] ba = TestBinarySortableSerDe.getRandBA(r, i);
-        MyTestClass t = new MyTestClass(b, s, n, l, f, d, st, bd, date, is, li, ba);
+        MyTestClass t = new MyTestClass();
+        ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
+        t.randomFill(r, extraTypeInfo);
         rows[i] = t;
       }