You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2012/07/12 18:52:40 UTC
svn commit: r1360812 - in /hive/trunk/serde/src:
java/org/apache/hadoop/hive/serde2/lazy/
java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/
java/org/apache/hadoop/hive/serde2/objectinspector/primitive/
test/org/apache/hadoop/hive/serde...
Author: hashutosh
Date: Thu Jul 12 16:52:40 2012
New Revision: 1360812
URL: http://svn.apache.org/viewvc?rev=1360812&view=rev
Log:
HIVE-3168: LazyBinaryObjectInspector.getPrimitiveJavaObject copies beyond length of underlying BytesWritable (Thejas Nair via Ashutosh Chauhan)
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBinaryObjectInspector.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBinaryObjectInspector.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBinaryObjectInspector.java
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java?rev=1360812&r1=1360811&r2=1360812&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java Thu Jul 12 16:52:40 2012
@@ -349,6 +349,20 @@ public final class LazyUtils {
}
}
+ /**
+ * Creates a ByteArrayRef with data from source BytesWritable
+ * @param sourceBw - source BytesWritable
+ */
+ public static ByteArrayRef createByteArrayRef(BytesWritable sourceBw){
+ //TODO should use BytesWritable.copyData() here once Hive
+ // removes support for the Hadoop 0.20 series.
+ byte[] newData = Arrays.copyOf(sourceBw.getBytes(), sourceBw.getLength());
+
+ ByteArrayRef bar = new ByteArrayRef();
+ bar.setData(newData);
+ return bar;
+ }
+
private LazyUtils() {
// prevent instantiation
}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBinaryObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBinaryObjectInspector.java?rev=1360812&r1=1360811&r2=1360812&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBinaryObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBinaryObjectInspector.java Thu Jul 12 16:52:40 2012
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.serde2.la
import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
import org.apache.hadoop.hive.serde2.lazy.LazyBinary;
+import org.apache.hadoop.hive.serde2.lazy.LazyUtils;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.hadoop.io.BytesWritable;
@@ -42,9 +43,7 @@ public class LazyBinaryObjectInspector e
if (null == o) {
return null;
}
- ByteArrayRef ba = new ByteArrayRef();
- ba.setData(((LazyBinary) o).getWritableObject().getBytes());
- return ba;
+ return LazyUtils.createByteArrayRef(((LazyBinary) o).getWritableObject());
}
@Override
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBinaryObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBinaryObjectInspector.java?rev=1360812&r1=1360811&r2=1360812&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBinaryObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaBinaryObjectInspector.java Thu Jul 12 16:52:40 2012
@@ -1,6 +1,7 @@
package org.apache.hadoop.hive.serde2.objectinspector.primitive;
import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
+import org.apache.hadoop.hive.serde2.lazy.LazyUtils;
import org.apache.hadoop.io.BytesWritable;
/**
@@ -63,9 +64,7 @@ public class JavaBinaryObjectInspector e
if (null == bw){
return null;
}
- ByteArrayRef ba = (ByteArrayRef)o;
- ba.setData(bw.getBytes());
- return ba;
+ return LazyUtils.createByteArrayRef(bw);
}
@Override
@@ -80,9 +79,7 @@ public class JavaBinaryObjectInspector e
if(null == bw){
return null;
}
- ByteArrayRef ba = new ByteArrayRef();
- ba.setData(bw.getBytes());
- return ba;
+ return LazyUtils.createByteArrayRef(bw);
}
}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBinaryObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBinaryObjectInspector.java?rev=1360812&r1=1360811&r2=1360812&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBinaryObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableBinaryObjectInspector.java Thu Jul 12 16:52:40 2012
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.serde2.objectinspector.primitive;
import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
+import org.apache.hadoop.hive.serde2.lazy.LazyUtils;
import org.apache.hadoop.io.BytesWritable;
public class WritableBinaryObjectInspector extends AbstractPrimitiveWritableObjectInspector
@@ -44,9 +45,7 @@ public class WritableBinaryObjectInspect
if (null == o){
return null;
}
- ByteArrayRef ba = new ByteArrayRef();
- ba.setData(((BytesWritable)o).getBytes());
- return ba;
+ return LazyUtils.createByteArrayRef((BytesWritable)o);
}
@Override
Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java?rev=1360812&r1=1360811&r2=1360812&view=diff
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java (original)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazybinary/TestLazyBinarySerDe.java Thu Jul 12 16:52:40 2012
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.serde2.lazybinary;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
@@ -35,14 +36,22 @@ import org.apache.hadoop.hive.serde2.bin
import org.apache.hadoop.hive.serde2.binarysortable.MyTestInnerStruct;
import org.apache.hadoop.hive.serde2.binarysortable.TestBinarySortableSerDe;
import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
+import org.apache.hadoop.hive.serde2.lazy.LazyBinary;
+import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.AbstractPrimitiveLazyObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyPrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryMapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaBinaryObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBinaryObjectInspector;
import org.apache.hadoop.io.BytesWritable;
/**
@@ -53,7 +62,7 @@ public class TestLazyBinarySerDe extends
/**
* Generate a random struct array.
- *
+ *
* @param r
* random number generator
* @return an struct array
@@ -71,7 +80,7 @@ public class TestLazyBinarySerDe extends
/**
* Initialize the LazyBinarySerDe.
- *
+ *
* @param fieldNames
* table field names
* @param fieldTypes
@@ -91,7 +100,7 @@ public class TestLazyBinarySerDe extends
/**
* Test the LazyBinarySerDe.
- *
+ *
* @param rows
* array of structs to be serialized
* @param rowOI
@@ -134,7 +143,7 @@ public class TestLazyBinarySerDe extends
* Compare two structs that have different number of fields. We just compare
* the first few common fields, ignoring the fields existing in one struct but
* not the other.
- *
+ *
* @see ObjectInspectorUtils#compare(Object, ObjectInspector, Object,
* ObjectInspector)
*/
@@ -464,7 +473,7 @@ public class TestLazyBinarySerDe extends
/**
* The test entrance function.
- *
+ *
* @throws Throwable
*/
public void testLazyBinarySerDe() throws Throwable {
@@ -522,4 +531,78 @@ public class TestLazyBinarySerDe extends
throw e;
}
}
+
+ private final byte[] inpBArray = {'1','\u0001','3','4'};
+ private BytesWritable getInputBytesWritable() {
+ //create input BytesWritable. This would have capacity greater than length)
+ BytesWritable bW = new BytesWritable();
+ bW.set(inpBArray, 0, inpBArray.length);
+ return bW;
+ }
+
+ /**
+ * Test to see if ByteArrayRef with correct contents is generated by
+ * JavaBinaryObjectInspector from input BytesWritable
+ * @throws Throwable
+ */
+ public void testJavaBinaryObjectInspector() throws Throwable {
+ BytesWritable bW = getInputBytesWritable();
+
+ //create JavaBinaryObjectInspector
+ JavaBinaryObjectInspector binInspector =
+ PrimitiveObjectInspectorFactory.javaByteArrayObjectInspector;
+
+ //convert BytesWritable to ByteArrayRef
+ ByteArrayRef outBARef = binInspector.set(null, bW);
+
+ assertTrue("compare input and output BAs",
+ Arrays.equals(inpBArray, outBARef.getData()));
+ }
+
+
+ /**
+ * Test to see if ByteArrayRef with correct contents is generated by
+ * WritableBinaryObjectInspector from input BytesWritable
+ * @throws Throwable
+ */
+ public void testWritableBinaryObjectInspector() throws Throwable {
+ BytesWritable bW = getInputBytesWritable();
+
+ //test WritableBinaryObjectInspector
+ WritableBinaryObjectInspector writableBinInsp =
+ PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
+
+ //convert BytesWritable to ByteArrayRef
+ ByteArrayRef outBARef = writableBinInsp.getPrimitiveJavaObject(bW);
+
+ assertTrue("compare input and output BAs",
+ Arrays.equals(inpBArray, outBARef.getData()));
+ }
+
+ /**
+ * Test to see if ByteArrayRef with correct contents is generated by
+ * LazyBinaryObjectInspector from input BytesWritable
+ * @throws Throwable
+ */
+ public void testLazyBinaryObjectInspector() throws Throwable {
+
+ //create input ByteArrayRef
+ ByteArrayRef inpBARef = new ByteArrayRef();
+ inpBARef.setData(inpBArray);
+
+ AbstractPrimitiveLazyObjectInspector<?> binInspector = LazyPrimitiveObjectInspectorFactory
+ .getLazyObjectInspector(PrimitiveCategory.BINARY, false, (byte)0);
+
+ //create LazyBinary initialed with inputBA
+ LazyBinary lazyBin = (LazyBinary) LazyFactory.createLazyObject(binInspector);
+ lazyBin.init(inpBARef, 0, inpBArray.length);
+
+ //use inspector to get a ByteArrayRef out of LazyBinary
+ ByteArrayRef outBARef = (ByteArrayRef) binInspector.getPrimitiveJavaObject(lazyBin);
+
+ assertTrue("compare input and output BAs",
+ Arrays.equals(inpBArray, outBARef.getData()));
+
+ }
+
}