You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by zs...@apache.org on 2010/01/21 10:00:43 UTC
svn commit: r901604 [1/2] - in
/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy: ./
objectinspector/ objectinspector/primitive/
Author: zshao
Date: Thu Jan 21 08:59:22 2010
New Revision: 901604
URL: http://svn.apache.org/viewvc?rev=901604&view=rev
Log:
HIVE-1081. Automated source code cleanup - Part 2 - serde. (Carl Steinbach via zshao)
Modified:
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/ByteArrayRef.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBoolean.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyNonPrimitive.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyListObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyMapObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/AbstractPrimitiveLazyObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBooleanObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyByteObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyDoubleObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyFloatObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyIntObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyLongObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyShortObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyStringObjectInspector.java
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyVoidObjectInspector.java
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/ByteArrayRef.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/ByteArrayRef.java?rev=901604&r1=901603&r2=901604&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/ByteArrayRef.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/ByteArrayRef.java Thu Jan 21 08:59:22 2010
@@ -20,14 +20,12 @@
/**
* ByteArrayRef stores a reference to a byte array.
*
- * The LazyObject hierarchy uses a reference to a single ByteArrayRef,
- * so that it's much faster to switch to the next row and release the
- * reference to the old row (so that the system can do garbage collection
- * if needed).
+ * The LazyObject hierarchy uses a reference to a single ByteArrayRef, so that
+ * it's much faster to switch to the next row and release the reference to the
+ * old row (so that the system can do garbage collection if needed).
*/
public class ByteArrayRef {
-
/**
* Stores the actual data.
*/
@@ -40,5 +38,5 @@
public void setData(byte[] data) {
this.data = data;
}
-
+
}
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java?rev=901604&r1=901603&r2=901604&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java Thu Jan 21 08:59:22 2010
@@ -28,8 +28,8 @@
/**
* LazyArray stores an array of Lazy Objects.
*
- * LazyArray does not deal with the case of a NULL array. That is handled
- * by the parent LazyObject.
+ * LazyArray does not deal with the case of a NULL array. That is handled by the
+ * parent LazyObject.
*/
public class LazyArray extends LazyNonPrimitive<LazyListObjectInspector> {
@@ -38,36 +38,36 @@
*/
boolean parsed = false;
/**
- * The length of the array.
- * Only valid when the data is parsed.
- * -1 when the array is NULL.
+ * The length of the array. Only valid when the data is parsed. -1 when the
+ * array is NULL.
*/
int arrayLength = 0;
-
+
/**
- * The start positions of array elements.
- * Only valid when the data is parsed.
- * Note that startPosition[arrayLength] = begin + length + 1;
- * that makes sure we can use the same formula to compute the
- * length of each element of the array.
+ * The start positions of array elements. Only valid when the data is parsed.
+ * Note that startPosition[arrayLength] = begin + length + 1; that makes sure
+ * we can use the same formula to compute the length of each element of the
+ * array.
*/
int[] startPosition;
-
+
/**
* Whether init() has been called on the element or not.
*/
boolean[] elementInited;
-
+
/**
- * The elements of the array. Note that we do arrayElements[i].
- * init(bytes, begin, length) only when that element is accessed.
+ * The elements of the array. Note that we do arrayElements[i]. init(bytes,
+ * begin, length) only when that element is accessed.
*/
LazyObject[] arrayElements;
/**
* Construct a LazyArray object with the ObjectInspector.
- * @param oi the oi representing the type of this LazyArray as well as meta
- * information like separator etc.
+ *
+ * @param oi
+ * the oi representing the type of this LazyArray as well as meta
+ * information like separator etc.
*/
protected LazyArray(LazyListObjectInspector oi) {
super(oi);
@@ -75,6 +75,7 @@
/**
* Set the row data for this LazyArray.
+ *
* @see LazyObject#init(ByteArrayRef, int, int)
*/
@Override
@@ -82,84 +83,83 @@
super.init(bytes, start, length);
parsed = false;
}
-
+
/**
- * Enlarge the size of arrays storing information for the elements inside
- * the array.
+ * Enlarge the size of arrays storing information for the elements inside the
+ * array.
*/
private void enlargeArrays() {
if (startPosition == null) {
int initialSize = 2;
- startPosition = new int[initialSize];
+ startPosition = new int[initialSize];
arrayElements = new LazyObject[initialSize];
elementInited = new boolean[initialSize];
} else {
- startPosition = Arrays.copyOf(startPosition, startPosition.length*2);
- arrayElements = Arrays.copyOf(arrayElements, arrayElements.length*2);
- elementInited = Arrays.copyOf(elementInited, elementInited.length*2);
+ startPosition = Arrays.copyOf(startPosition, startPosition.length * 2);
+ arrayElements = Arrays.copyOf(arrayElements, arrayElements.length * 2);
+ elementInited = Arrays.copyOf(elementInited, elementInited.length * 2);
}
}
-
+
/**
* Parse the bytes and fill arrayLength and startPosition.
*/
private void parse() {
parsed = true;
-
+
byte separator = oi.getSeparator();
boolean isEscaped = oi.isEscaped();
byte escapeChar = oi.getEscapeChar();
-
+
// empty array?
if (length == 0) {
arrayLength = 0;
return;
}
-
+
byte[] bytes = this.bytes.getData();
-
+
arrayLength = 0;
int arrayByteEnd = start + length;
int elementByteBegin = start;
int elementByteEnd = start;
-
+
// Go through all bytes in the byte[]
while (elementByteEnd <= arrayByteEnd) {
// Reached the end of a field?
- if (elementByteEnd == arrayByteEnd
- || bytes[elementByteEnd] == separator) {
+ if (elementByteEnd == arrayByteEnd || bytes[elementByteEnd] == separator) {
// Array size not big enough?
- if (startPosition == null || arrayLength+1 == startPosition.length) {
+ if (startPosition == null || arrayLength + 1 == startPosition.length) {
enlargeArrays();
}
startPosition[arrayLength] = elementByteBegin;
arrayLength++;
elementByteBegin = elementByteEnd + 1;
- elementByteEnd ++;
+ elementByteEnd++;
} else {
if (isEscaped && bytes[elementByteEnd] == escapeChar
- && elementByteEnd+1 < arrayByteEnd) {
+ && elementByteEnd + 1 < arrayByteEnd) {
// ignore the char after escape_char
elementByteEnd += 2;
} else {
- elementByteEnd ++;
+ elementByteEnd++;
}
- }
+ }
}
// Store arrayByteEnd+1 in startPosition[arrayLength]
// so that we can use the same formula to compute the length of
// each element in the array: startPosition[i+1] - startPosition[i] - 1
startPosition[arrayLength] = arrayByteEnd + 1;
-
+
if (arrayLength > 0) {
Arrays.fill(elementInited, 0, arrayLength, false);
}
-
+
}
-
+
/**
- * Returns the actual primitive object at the index position
- * inside the array represented by this LazyObject.
+ * Returns the actual primitive object at the index position inside the array
+ * represented by this LazyObject.
*/
public Object getListElementObject(int index) {
if (!parsed) {
@@ -170,34 +170,35 @@
}
return uncheckedGetElement(index);
}
-
+
/**
* Get the element without checking out-of-bound index.
*/
private Object uncheckedGetElement(int index) {
Text nullSequence = oi.getNullSequence();
- int elementLength = startPosition[index+1] - startPosition[index] - 1;
- if (elementLength == nullSequence.getLength()
- && 0 == LazyUtils.compare(bytes.getData(), startPosition[index],
- elementLength, nullSequence.getBytes(), 0,
- nullSequence.getLength())) {
+ int elementLength = startPosition[index + 1] - startPosition[index] - 1;
+ if (elementLength == nullSequence.getLength()
+ && 0 == LazyUtils
+ .compare(bytes.getData(), startPosition[index], elementLength,
+ nullSequence.getBytes(), 0, nullSequence.getLength())) {
return null;
} else {
if (!elementInited[index]) {
elementInited[index] = true;
if (arrayElements[index] == null) {
- arrayElements[index] = LazyFactory.createLazyObject(
- ((ListObjectInspector)oi).getListElementObjectInspector());
+ arrayElements[index] = LazyFactory
+ .createLazyObject(((ListObjectInspector) oi)
+ .getListElementObjectInspector());
}
- arrayElements[index].init(bytes, startPosition[index],
- elementLength);
+ arrayElements[index].init(bytes, startPosition[index], elementLength);
}
}
return arrayElements[index].getObject();
}
-
- /** Returns -1 for null array.
+
+ /**
+ * Returns -1 for null array.
*/
public int getListLength() {
if (!parsed) {
@@ -205,14 +206,15 @@
}
return arrayLength;
}
-
- /**
- * cachedList is reused every time getList is called.
- * Different LazyArray instances cannot share the same cachedList.
+
+ /**
+ * cachedList is reused every time getList is called. Different LazyArray
+ * instances cannot share the same cachedList.
*/
ArrayList<Object> cachedList;
- /** Returns the List of actual primitive objects.
- * Returns null for null array.
+
+ /**
+ * Returns the List of actual primitive objects. Returns null for null array.
*/
public List<Object> getList() {
if (!parsed) {
@@ -226,7 +228,7 @@
} else {
cachedList.clear();
}
- for (int index=0; index<arrayLength; index++) {
+ for (int index = 0; index < arrayLength; index++) {
cachedList.add(uncheckedGetElement(index));
}
return cachedList;
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBoolean.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBoolean.java?rev=901604&r1=901603&r2=901604&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBoolean.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBoolean.java Thu Jan 21 08:59:22 2010
@@ -33,7 +33,8 @@
* </p>
*
*/
-public class LazyBoolean extends LazyPrimitive<LazyBooleanObjectInspector, BooleanWritable> {
+public class LazyBoolean extends
+ LazyPrimitive<LazyBooleanObjectInspector, BooleanWritable> {
public LazyBoolean(LazyBooleanObjectInspector oi) {
super(oi);
@@ -44,28 +45,26 @@
super(copy);
data = new BooleanWritable(copy.data.get());
}
-
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
- if (length == 4
- && Character.toUpperCase(bytes.getData()[start]) == 'T'
- && Character.toUpperCase(bytes.getData()[start+1]) == 'R'
- && Character.toUpperCase(bytes.getData()[start+2]) == 'U'
- && Character.toUpperCase(bytes.getData()[start+3]) == 'E') {
+ if (length == 4 && Character.toUpperCase(bytes.getData()[start]) == 'T'
+ && Character.toUpperCase(bytes.getData()[start + 1]) == 'R'
+ && Character.toUpperCase(bytes.getData()[start + 2]) == 'U'
+ && Character.toUpperCase(bytes.getData()[start + 3]) == 'E') {
data.set(true);
isNull = false;
} else if (length == 5
- && Character.toUpperCase(bytes.getData()[start]) == 'F'
- && Character.toUpperCase(bytes.getData()[start+1]) == 'A'
- && Character.toUpperCase(bytes.getData()[start+2]) == 'L'
- && Character.toUpperCase(bytes.getData()[start+3]) == 'S'
- && Character.toUpperCase(bytes.getData()[start+4]) == 'E') {
+ && Character.toUpperCase(bytes.getData()[start]) == 'F'
+ && Character.toUpperCase(bytes.getData()[start + 1]) == 'A'
+ && Character.toUpperCase(bytes.getData()[start + 2]) == 'L'
+ && Character.toUpperCase(bytes.getData()[start + 3]) == 'S'
+ && Character.toUpperCase(bytes.getData()[start + 4]) == 'E') {
data.set(false);
isNull = false;
- } else {
+ } else {
isNull = true;
}
}
-
}
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java?rev=901604&r1=901603&r2=901604&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java Thu Jan 21 08:59:22 2010
@@ -33,7 +33,8 @@
* </p>
*
*/
-public class LazyByte extends LazyPrimitive<LazyByteObjectInspector, ByteWritable> {
+public class LazyByte extends
+ LazyPrimitive<LazyByteObjectInspector, ByteWritable> {
public LazyByte(LazyByteObjectInspector oi) {
super(oi);
@@ -44,7 +45,7 @@
super(copy);
data = new ByteWritable(copy.data.get());
}
-
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
try {
@@ -54,7 +55,7 @@
isNull = true;
}
}
-
+
/**
* Parses the string argument as if it was a byte value and returns the
* result. Throws NumberFormatException if the string does not represent a
@@ -63,15 +64,16 @@
* @param bytes
* @param start
* @param length
- * a UTF-8 encoded string representation of a single byte quantity.
+ * a UTF-8 encoded string representation of a single byte quantity.
* @return byte the value represented by the argument
* @throws NumberFormatException
- * if the argument could not be parsed as a byte quantity.
+ * if the argument could not be parsed as a byte quantity.
*/
- public static byte parseByte(byte[] bytes, int start, int length) throws NumberFormatException {
- return parseByte(bytes, start, length, 10);
+ public static byte parseByte(byte[] bytes, int start, int length)
+ throws NumberFormatException {
+ return parseByte(bytes, start, length, 10);
}
-
+
/**
* Parses the string argument as if it was a byte value and returns the
* result. Throws NumberFormatException if the string does not represent a
@@ -81,22 +83,21 @@
* @param bytes
* @param start
* @param length
- * a UTF-8 encoded string representation of a single byte quantity.
+ * a UTF-8 encoded string representation of a single byte quantity.
* @param radix
- * the radix to use when parsing.
+ * the radix to use when parsing.
* @return byte the value represented by the argument
* @throws NumberFormatException
- * if the argument could not be parsed as a byte quantity.
+ * if the argument could not be parsed as a byte quantity.
*/
public static byte parseByte(byte[] bytes, int start, int length, int radix)
- throws NumberFormatException {
- int intValue = LazyInteger.parseInt(bytes, start, length, radix);
- byte result = (byte) intValue;
- if (result == intValue) {
- return result;
- }
- throw new NumberFormatException();
+ throws NumberFormatException {
+ int intValue = LazyInteger.parseInt(bytes, start, length, radix);
+ byte result = (byte) intValue;
+ if (result == intValue) {
+ return result;
+ }
+ throw new NumberFormatException();
}
-
}
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java?rev=901604&r1=901603&r2=901604&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java Thu Jan 21 08:59:22 2010
@@ -27,7 +27,8 @@
* LazyObject for storing a value of Double.
*
*/
-public class LazyDouble extends LazyPrimitive<LazyDoubleObjectInspector, DoubleWritable> {
+public class LazyDouble extends
+ LazyPrimitive<LazyDoubleObjectInspector, DoubleWritable> {
public LazyDouble(LazyDoubleObjectInspector oi) {
super(oi);
@@ -37,8 +38,8 @@
public LazyDouble(LazyDouble copy) {
super(copy);
data = new DoubleWritable(copy.data.get());
- }
-
+ }
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
try {
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java?rev=901604&r1=901603&r2=901604&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java Thu Jan 21 08:59:22 2010
@@ -37,57 +37,49 @@
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableByteObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDoubleObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableFloatObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableIntObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableLongObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableShortObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.io.Text;
public class LazyFactory {
/**
- * Create a lazy primitive class given the type name.
+ * Create a lazy primitive class given the type name.
*/
- public static LazyPrimitive<?,?> createLazyPrimitiveClass(PrimitiveObjectInspector oi) {
+ public static LazyPrimitive<?, ?> createLazyPrimitiveClass(
+ PrimitiveObjectInspector oi) {
PrimitiveCategory p = oi.getPrimitiveCategory();
- switch(p) {
- case BOOLEAN: {
- return new LazyBoolean((LazyBooleanObjectInspector)oi);
- }
- case BYTE: {
- return new LazyByte((LazyByteObjectInspector)oi);
- }
- case SHORT: {
- return new LazyShort((LazyShortObjectInspector)oi);
- }
- case INT: {
- return new LazyInteger((LazyIntObjectInspector)oi);
- }
- case LONG: {
- return new LazyLong((LazyLongObjectInspector)oi);
- }
- case FLOAT: {
- return new LazyFloat((LazyFloatObjectInspector)oi);
- }
- case DOUBLE: {
- return new LazyDouble((LazyDoubleObjectInspector)oi);
- }
- case STRING: {
- return new LazyString((LazyStringObjectInspector)oi);
- }
- default: {
- throw new RuntimeException("Internal error: no LazyObject for " + p);
- }
+ switch (p) {
+ case BOOLEAN: {
+ return new LazyBoolean((LazyBooleanObjectInspector) oi);
+ }
+ case BYTE: {
+ return new LazyByte((LazyByteObjectInspector) oi);
+ }
+ case SHORT: {
+ return new LazyShort((LazyShortObjectInspector) oi);
+ }
+ case INT: {
+ return new LazyInteger((LazyIntObjectInspector) oi);
+ }
+ case LONG: {
+ return new LazyLong((LazyLongObjectInspector) oi);
+ }
+ case FLOAT: {
+ return new LazyFloat((LazyFloatObjectInspector) oi);
+ }
+ case DOUBLE: {
+ return new LazyDouble((LazyDoubleObjectInspector) oi);
+ }
+ case STRING: {
+ return new LazyString((LazyStringObjectInspector) oi);
+ }
+ default: {
+ throw new RuntimeException("Internal error: no LazyObject for " + p);
+ }
}
}
@@ -96,70 +88,75 @@
*/
public static LazyObject createLazyObject(ObjectInspector oi) {
ObjectInspector.Category c = oi.getCategory();
- switch(c) {
+ switch (c) {
case PRIMITIVE:
- return createLazyPrimitiveClass((PrimitiveObjectInspector)oi);
+ return createLazyPrimitiveClass((PrimitiveObjectInspector) oi);
case MAP:
- return new LazyMap((LazyMapObjectInspector)oi);
- case LIST:
- return new LazyArray((LazyListObjectInspector)oi);
+ return new LazyMap((LazyMapObjectInspector) oi);
+ case LIST:
+ return new LazyArray((LazyListObjectInspector) oi);
case STRUCT:
- return new LazyStruct((LazySimpleStructObjectInspector)oi);
+ return new LazyStruct((LazySimpleStructObjectInspector) oi);
}
throw new RuntimeException("Hive LazySerDe Internal error.");
}
-
+
/**
* Create a hierarchical ObjectInspector for LazyObject with the given
* typeInfo.
- * @param typeInfo The type information for the LazyObject
- * @param separator The array of separators for delimiting each level
- * @param separatorIndex The current level (for separators). List(array),
- * struct uses 1 level of separator, and map uses 2
- * levels: the first one for delimiting entries, the
- * second one for delimiting key and values.
- * @param nullSequence The sequence of bytes representing NULL.
- * @return The ObjectInspector
+ *
+ * @param typeInfo
+ * The type information for the LazyObject
+ * @param separator
+ * The array of separators for delimiting each level
+ * @param separatorIndex
+ * The current level (for separators). List(array), struct uses 1
+ * level of separator, and map uses 2 levels: the first one for
+ * delimiting entries, the second one for delimiting key and values.
+ * @param nullSequence
+ * The sequence of bytes representing NULL.
+ * @return The ObjectInspector
*/
- public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, byte[] separator,
- int separatorIndex, Text nullSequence, boolean escaped, byte escapeChar) {
+ public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo,
+ byte[] separator, int separatorIndex, Text nullSequence, boolean escaped,
+ byte escapeChar) {
ObjectInspector.Category c = typeInfo.getCategory();
- switch(c) {
+ switch (c) {
case PRIMITIVE:
return LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(
- ((PrimitiveTypeInfo)typeInfo).getPrimitiveCategory(), escaped, escapeChar);
+ ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory(), escaped,
+ escapeChar);
case MAP:
return LazyObjectInspectorFactory.getLazySimpleMapObjectInspector(
- createLazyObjectInspector(((MapTypeInfo)typeInfo).getMapKeyTypeInfo(),
- separator, separatorIndex+2, nullSequence, escaped, escapeChar),
- createLazyObjectInspector(((MapTypeInfo)typeInfo).getMapValueTypeInfo(),
- separator, separatorIndex+2, nullSequence, escaped, escapeChar),
- separator[separatorIndex],
- separator[separatorIndex+1],
+ createLazyObjectInspector(((MapTypeInfo) typeInfo)
+ .getMapKeyTypeInfo(), separator, separatorIndex + 2,
+ nullSequence, escaped, escapeChar), createLazyObjectInspector(
+ ((MapTypeInfo) typeInfo).getMapValueTypeInfo(), separator,
+ separatorIndex + 2, nullSequence, escaped, escapeChar),
+ separator[separatorIndex], separator[separatorIndex + 1],
nullSequence, escaped, escapeChar);
- case LIST:
+ case LIST:
return LazyObjectInspectorFactory.getLazySimpleListObjectInspector(
- createLazyObjectInspector(((ListTypeInfo)typeInfo).getListElementTypeInfo(),
- separator, separatorIndex+1, nullSequence, escaped, escapeChar),
- separator[separatorIndex],
+ createLazyObjectInspector(((ListTypeInfo) typeInfo)
+ .getListElementTypeInfo(), separator, separatorIndex + 1,
+ nullSequence, escaped, escapeChar), separator[separatorIndex],
nullSequence, escaped, escapeChar);
case STRUCT:
- StructTypeInfo structTypeInfo = (StructTypeInfo)typeInfo;
+ StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
List<String> fieldNames = structTypeInfo.getAllStructFieldNames();
- List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos();
- List<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>(fieldTypeInfos.size());
- for(int i=0; i<fieldTypeInfos.size(); i++) {
- fieldObjectInspectors.add(
- createLazyObjectInspector(fieldTypeInfos.get(i), separator,
- separatorIndex+1, nullSequence, escaped, escapeChar));
+ List<TypeInfo> fieldTypeInfos = structTypeInfo
+ .getAllStructFieldTypeInfos();
+ List<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>(
+ fieldTypeInfos.size());
+ for (int i = 0; i < fieldTypeInfos.size(); i++) {
+ fieldObjectInspectors.add(createLazyObjectInspector(fieldTypeInfos
+ .get(i), separator, separatorIndex + 1, nullSequence, escaped,
+ escapeChar));
}
return LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(
- fieldNames,
- fieldObjectInspectors,
- separator[separatorIndex],
- nullSequence,
- false, escaped, escapeChar);
+ fieldNames, fieldObjectInspectors, separator[separatorIndex],
+ nullSequence, false, escaped, escapeChar);
}
throw new RuntimeException("Hive LazySerDe Internal error.");
@@ -169,29 +166,33 @@
* Create a hierarchical ObjectInspector for LazyStruct with the given
* columnNames and columnTypeInfos.
*
- * @param lastColumnTakesRest whether the last column of the struct should take
- * the rest of the row if there are extra fields.
- * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text, boolean, byte)
- */
- public static ObjectInspector createLazyStructInspector(List<String> columnNames,
- List<TypeInfo> typeInfos, byte[] separators,
- Text nullSequence, boolean lastColumnTakesRest, boolean escaped, byte escapeChar) {
- ArrayList<ObjectInspector> columnObjectInspectors =
- new ArrayList<ObjectInspector>(typeInfos.size());
- for (int i=0; i<typeInfos.size(); i++) {
- columnObjectInspectors.add(
- LazyFactory.createLazyObjectInspector(typeInfos.get(i), separators, 1, nullSequence, escaped, escapeChar));
- }
- return
- LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(columnNames,
- columnObjectInspectors, separators[0], nullSequence, lastColumnTakesRest, escaped, escapeChar);
+ * @param lastColumnTakesRest
+ * whether the last column of the struct should take the rest of the
+ * row if there are extra fields.
+ * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text,
+ * boolean, byte)
+ */
+ public static ObjectInspector createLazyStructInspector(
+ List<String> columnNames, List<TypeInfo> typeInfos, byte[] separators,
+ Text nullSequence, boolean lastColumnTakesRest, boolean escaped,
+ byte escapeChar) {
+ ArrayList<ObjectInspector> columnObjectInspectors = new ArrayList<ObjectInspector>(
+ typeInfos.size());
+ for (int i = 0; i < typeInfos.size(); i++) {
+ columnObjectInspectors.add(LazyFactory.createLazyObjectInspector(
+ typeInfos.get(i), separators, 1, nullSequence, escaped, escapeChar));
+ }
+ return LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(
+ columnNames, columnObjectInspectors, separators[0], nullSequence,
+ lastColumnTakesRest, escaped, escapeChar);
}
-
+
/**
* Create a hierarchical ObjectInspector for ColumnarStruct with the given
* columnNames and columnTypeInfos.
*
- * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text, boolean, byte)
+ * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text,
+ * boolean, byte)
*/
public static ObjectInspector createColumnarStructInspector(
List<String> columnNames, List<TypeInfo> columnTypes, byte[] separators,
@@ -199,11 +200,12 @@
ArrayList<ObjectInspector> columnObjectInspectors = new ArrayList<ObjectInspector>(
columnTypes.size());
for (int i = 0; i < columnTypes.size(); i++) {
- columnObjectInspectors.add(LazyFactory.createLazyObjectInspector(
- columnTypes.get(i), separators, 1, nullSequence, escaped, escapeChar));
+ columnObjectInspectors
+ .add(LazyFactory.createLazyObjectInspector(columnTypes.get(i),
+ separators, 1, nullSequence, escaped, escapeChar));
}
return ObjectInspectorFactory.getColumnarStructObjectInspector(columnNames,
columnObjectInspectors, nullSequence);
}
-
+
}
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java?rev=901604&r1=901603&r2=901604&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java Thu Jan 21 08:59:22 2010
@@ -27,18 +27,19 @@
* LazyObject for storing a value of Double.
*
*/
-public class LazyFloat extends LazyPrimitive<LazyFloatObjectInspector, FloatWritable> {
+public class LazyFloat extends
+ LazyPrimitive<LazyFloatObjectInspector, FloatWritable> {
public LazyFloat(LazyFloatObjectInspector oi) {
super(oi);
data = new FloatWritable();
}
-
+
public LazyFloat(LazyFloat copy) {
super(copy);
data = new FloatWritable(copy.data.get());
- }
-
+ }
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
try {
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java?rev=901604&r1=901603&r2=901604&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java Thu Jan 21 08:59:22 2010
@@ -36,7 +36,8 @@
* </p>
*
*/
-public class LazyInteger extends LazyPrimitive<LazyIntObjectInspector, IntWritable> {
+public class LazyInteger extends
+ LazyPrimitive<LazyIntObjectInspector, IntWritable> {
public LazyInteger(LazyIntObjectInspector oi) {
super(oi);
@@ -46,9 +47,8 @@
public LazyInteger(LazyInteger copy) {
super(copy);
data = new IntWritable(copy.data.get());
- }
-
-
+ }
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
try {
@@ -58,7 +58,7 @@
isNull = true;
}
}
-
+
/**
* Parses the string argument as if it was an int value and returns the
* result. Throws NumberFormatException if the string does not represent an
@@ -67,13 +67,14 @@
* @param bytes
* @param start
* @param length
- * a UTF-8 encoded string representation of an int quantity.
+ * a UTF-8 encoded string representation of an int quantity.
* @return int the value represented by the argument
* @exception NumberFormatException
- * if the argument could not be parsed as an int quantity.
+ * if the argument could not be parsed as an int quantity.
*/
- public static int parseInt(byte[] bytes, int start, int length) throws NumberFormatException {
- return parseInt(bytes, start, length, 10);
+ public static int parseInt(byte[] bytes, int start, int length)
+ throws NumberFormatException {
+ return parseInt(bytes, start, length, 10);
}
/**
@@ -85,20 +86,19 @@
* @param bytes
* @param start
* @param length
- * a UTF-8 encoded string representation of an int quantity.
+ * a UTF-8 encoded string representation of an int quantity.
* @param radix
- * the base to use for conversion.
- * @return the value represented by the argument
+ * the base to use for conversion.
+ * @return the value represented by the argument
* @exception NumberFormatException
- * if the argument could not be parsed as an int quantity.
+ * if the argument could not be parsed as an int quantity.
*/
public static int parseInt(byte[] bytes, int start, int length, int radix)
- throws NumberFormatException {
+ throws NumberFormatException {
if (bytes == null) {
throw new NumberFormatException("String is null");
}
- if (radix < Character.MIN_RADIX ||
- radix > Character.MAX_RADIX) {
+ if (radix < Character.MIN_RADIX || radix > Character.MAX_RADIX) {
throw new NumberFormatException("Invalid radix: " + radix);
}
if (length == 0) {
@@ -109,7 +109,8 @@
if (negative || bytes[start] == '+') {
offset++;
if (length == 1) {
- throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
+ length));
}
}
@@ -121,57 +122,63 @@
* @param bytes
* @param start
* @param length
- * a UTF-8 encoded string representation of an int quantity.
+ * a UTF-8 encoded string representation of an int quantity.
* @param radix
- * the base to use for conversion.
+ * the base to use for conversion.
* @param offset
- * the starting position after the sign (if exists)
+ * the starting position after the sign (if exists)
* @param radix
- * the base to use for conversion.
+ * the base to use for conversion.
* @param negative
- * whether the number is negative.
+ * whether the number is negative.
* @return the value represented by the argument
* @exception NumberFormatException
- * if the argument could not be parsed as an int quantity.
+ * if the argument could not be parsed as an int quantity.
*/
- private static int parse(byte[] bytes, int start, int length, int offset, int radix,
- boolean negative) throws NumberFormatException {
- int max = Integer.MIN_VALUE / radix;
- int result = 0, end = start + length;
- while (offset < end) {
- int digit = LazyUtils.digit(bytes[offset++], radix);
- if (digit == -1) {
- throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
- }
- if (max > result) {
- throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
- }
- int next = result * radix - digit;
- if (next > result) {
- throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
- }
- result = next;
+ private static int parse(byte[] bytes, int start, int length, int offset,
+ int radix, boolean negative) throws NumberFormatException {
+ int max = Integer.MIN_VALUE / radix;
+ int result = 0, end = start + length;
+ while (offset < end) {
+ int digit = LazyUtils.digit(bytes[offset++], radix);
+ if (digit == -1) {
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
+ length));
}
- if (!negative) {
- result = -result;
- if (result < 0) {
- throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
- }
+ if (max > result) {
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
+ length));
+ }
+ int next = result * radix - digit;
+ if (next > result) {
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
+ length));
+ }
+ result = next;
+ }
+ if (!negative) {
+ result = -result;
+ if (result < 0) {
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
+ length));
}
- return result;
+ }
+ return result;
}
/**
- * Writes out the text representation of an integer using base 10 to an
+ * Writes out the text representation of an integer using base 10 to an
* OutputStream in UTF-8 encoding.
- *
- * Note: division by a constant (like 10) is much faster than division by
- * a variable. That's one of the reasons that we don't make radix a
- * parameter here.
- *
- * @param out the outputstream to write to
- * @param i an int to write out
- * @throws IOException
+ *
+ * Note: division by a constant (like 10) is much faster than division by a
+ * variable. That's one of the reasons that we don't make radix a parameter
+ * here.
+ *
+ * @param out
+ * the outputstream to write to
+ * @param i
+ * an int to write out
+ * @throws IOException
*/
public static void writeUTF8(OutputStream out, int i) throws IOException {
if (i == 0) {
@@ -187,23 +194,23 @@
// of overflow here.
i = -i;
}
-
+
int start = 1000000000;
- while (i/start == 0) {
+ while (i / start == 0) {
start /= 10;
}
-
+
while (start > 0) {
out.write('0' - (i / start % 10));
start /= 10;
}
}
-
+
public static void writeUTF8NoException(OutputStream out, int i) {
try {
writeUTF8(out, i);
} catch (IOException e) {
throw new RuntimeException(e);
}
- }
+ }
}
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java?rev=901604&r1=901603&r2=901604&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java Thu Jan 21 08:59:22 2010
@@ -36,8 +36,8 @@
* </p>
*
*/
-public class LazyLong extends LazyPrimitive<LazyLongObjectInspector, LongWritable> {
-
+public class LazyLong extends
+ LazyPrimitive<LazyLongObjectInspector, LongWritable> {
public LazyLong(LazyLongObjectInspector oi) {
super(oi);
@@ -48,7 +48,7 @@
super(copy);
data = new LongWritable(copy.data.get());
}
-
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
try {
@@ -67,38 +67,38 @@
* @param bytes
* @param start
* @param length
- * a UTF-8 encoded string representation of a long quantity.
+ * a UTF-8 encoded string representation of a long quantity.
* @return long the value represented by the argument
* @exception NumberFormatException
- * if the argument could not be parsed as a long quantity.
+ * if the argument could not be parsed as a long quantity.
*/
- public static long parseLong(byte[] bytes, int start, int length) throws NumberFormatException {
+ public static long parseLong(byte[] bytes, int start, int length)
+ throws NumberFormatException {
return parseLong(bytes, start, length, 10);
}
/**
* Parses the string argument as if it was an long value and returns the
* result. Throws NumberFormatException if the string does not represent an
- * long quantity. The second argument specifies the radix to use when
- * parsing the value.
+ * long quantity. The second argument specifies the radix to use when parsing
+ * the value.
*
* @param bytes
* @param start
* @param length
- * a UTF-8 encoded string representation of a long quantity.
+ * a UTF-8 encoded string representation of a long quantity.
* @param radix
- * the base to use for conversion.
+ * the base to use for conversion.
* @return the value represented by the argument
* @exception NumberFormatException
- * if the argument could not be parsed as an long quantity.
+ * if the argument could not be parsed as an long quantity.
*/
public static long parseLong(byte[] bytes, int start, int length, int radix)
throws NumberFormatException {
if (bytes == null) {
throw new NumberFormatException("String is null");
}
- if (radix < Character.MIN_RADIX ||
- radix > Character.MAX_RADIX) {
+ if (radix < Character.MIN_RADIX || radix > Character.MAX_RADIX) {
throw new NumberFormatException("Invalid radix: " + radix);
}
if (length == 0) {
@@ -107,9 +107,10 @@
int offset = start;
boolean negative = bytes[start] == '-';
if (negative || bytes[start] == '+') {
- offset ++;
+ offset++;
if (length == 1) {
- throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
+ length));
}
}
@@ -117,61 +118,65 @@
}
/**
- /**
- * Parses the string argument as if it was an long value and returns the
+ * /** Parses the string argument as if it was an long value and returns the
* result. Throws NumberFormatException if the string does not represent an
- * long quantity. The second argument specifies the radix to use when
- * parsing the value.
+ * long quantity. The second argument specifies the radix to use when parsing
+ * the value.
*
* @param bytes
* @param start
* @param length
- * a UTF-8 encoded string representation of a long quantity.
+ * a UTF-8 encoded string representation of a long quantity.
* @param offset
- * the starting position after the sign (if exists)
+ * the starting position after the sign (if exists)
* @param radix
- * the base to use for conversion.
+ * the base to use for conversion.
* @param negative
- * whether the number is negative.
+ * whether the number is negative.
* @return the value represented by the argument
* @exception NumberFormatException
- * if the argument could not be parsed as an long quantity.
+ * if the argument could not be parsed as an long quantity.
*/
- private static long parse(byte[] bytes, int start, int length, int offset, int radix,
- boolean negative) {
+ private static long parse(byte[] bytes, int start, int length, int offset,
+ int radix, boolean negative) {
long max = Long.MIN_VALUE / radix;
long result = 0, end = start + length;
while (offset < end) {
int digit = LazyUtils.digit(bytes[offset++], radix);
if (digit == -1 || max > result) {
- throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
+ length));
}
long next = result * radix - digit;
if (next > result) {
- throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
+ length));
}
result = next;
}
if (!negative) {
result = -result;
if (result < 0) {
- throw new NumberFormatException(LazyUtils.convertToString(bytes, start, length));
+ throw new NumberFormatException(LazyUtils.convertToString(bytes, start,
+ length));
}
}
return result;
}
/**
- * Writes out the text representation of an integer using base 10 to an
+ * Writes out the text representation of an integer using base 10 to an
* OutputStream in UTF-8 encoding.
- *
- * Note: division by a constant (like 10) is much faster than division by
- * a variable. That's one of the reasons that we don't make radix a
- * parameter here.
- *
- * @param out the outputstream to write to
- * @param i an int to write out
- * @throws IOException
+ *
+ * Note: division by a constant (like 10) is much faster than division by a
+ * variable. That's one of the reasons that we don't make radix a parameter
+ * here.
+ *
+ * @param out
+ * the outputstream to write to
+ * @param i
+ * an int to write out
+ * @throws IOException
*/
public static void writeUTF8(OutputStream out, long i) throws IOException {
if (i == 0) {
@@ -187,14 +192,14 @@
// of overflow here.
i = -i;
}
-
+
long start = 1000000000000000000L;
- while (i/start == 0) {
+ while (i / start == 0) {
start /= 10;
}
-
+
while (start > 0) {
- out.write('0' - (int)((i / start) % 10));
+ out.write('0' - (int) ((i / start) % 10));
start /= 10;
}
}
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java?rev=901604&r1=901603&r2=901604&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java Thu Jan 21 08:59:22 2010
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.serde2.lazy;
import java.util.Arrays;
-import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
@@ -28,59 +27,55 @@
import org.apache.hadoop.io.Text;
/**
- * LazyMap stores a map of Primitive LazyObjects to LazyObjects.
- * Note that the keys of the map cannot contain null.
+ * LazyMap stores a map of Primitive LazyObjects to LazyObjects. Note that the
+ * keys of the map cannot contain null.
*
- * LazyMap does not deal with the case of a NULL map. That is handled
- * by the parent LazyObject.
+ * LazyMap does not deal with the case of a NULL map. That is handled by the
+ * parent LazyObject.
*/
public class LazyMap extends LazyNonPrimitive<LazyMapObjectInspector> {
-
+
/**
* Whether the data is already parsed or not.
*/
boolean parsed = false;
-
+
/**
- * The size of the map.
- * Only valid when the data is parsed.
- * -1 when the map is NULL.
+ * The size of the map. Only valid when the data is parsed. -1 when the map is
+ * NULL.
*/
int mapSize = 0;
-
+
/**
- * The beginning position of key[i].
- * Only valid when the data is parsed.
- * Note that keyStart[mapSize] = begin + length + 1;
- * that makes sure we can use the same formula to compute the
- * length of each value in the map.
+ * The beginning position of key[i]. Only valid when the data is parsed. Note
+ * that keyStart[mapSize] = begin + length + 1; that makes sure we can use the
+ * same formula to compute the length of each value in the map.
*/
int[] keyStart;
-
+
/**
- * The end position of key[i] (the position of the key-value separator).
- * Only valid when the data is parsed.
- */
+ * The end position of key[i] (the position of the key-value separator). Only
+ * valid when the data is parsed.
+ */
int[] keyEnd;
/**
* The keys are stored in an array of LazyPrimitives.
*/
- LazyPrimitive<?,?>[] keyObjects;
+ LazyPrimitive<?, ?>[] keyObjects;
/**
- * Whether init() is called on keyObjects[i].
+ * Whether init() is called on keyObjects[i].
*/
boolean[] keyInited;
/**
- * The values are stored in an array of LazyObjects.
- * value[index] will start from KeyEnd[index] + 1,
- * and ends before KeyStart[index+1] - 1.
+ * The values are stored in an array of LazyObjects. value[index] will start
+ * from KeyEnd[index] + 1, and ends before KeyStart[index+1] - 1.
*/
LazyObject[] valueObjects;
/**
* Whether init() is called on valueObjects[i]
*/
boolean[] valueInited;
-
+
/**
* Construct a LazyMap object with the ObjectInspector.
*/
@@ -90,6 +85,7 @@
/**
* Set the row data for this LazyArray.
+ *
* @see LazyObject#init(ByteArrayRef, int, int)
*/
@Override
@@ -97,27 +93,27 @@
super.init(bytes, start, length);
parsed = false;
}
-
+
/**
- * Enlarge the size of arrays storing information for the elements inside
- * the array.
+ * Enlarge the size of arrays storing information for the elements inside the
+ * array.
*/
protected void enlargeArrays() {
if (keyStart == null) {
int initialSize = 2;
keyStart = new int[initialSize];
keyEnd = new int[initialSize];
- keyObjects = new LazyPrimitive<?,?>[initialSize];
+ keyObjects = new LazyPrimitive<?, ?>[initialSize];
valueObjects = new LazyObject[initialSize];
keyInited = new boolean[initialSize];
valueInited = new boolean[initialSize];
} else {
- keyStart = Arrays.copyOf(keyStart, keyStart.length*2);
- keyEnd = Arrays.copyOf(keyEnd, keyEnd.length*2);
- keyObjects = Arrays.copyOf(keyObjects, keyObjects.length*2);
- valueObjects = Arrays.copyOf(valueObjects, valueObjects.length*2);
- keyInited = Arrays.copyOf(keyInited, keyInited.length*2);
- valueInited = Arrays.copyOf(valueInited, valueInited.length*2);
+ keyStart = Arrays.copyOf(keyStart, keyStart.length * 2);
+ keyEnd = Arrays.copyOf(keyEnd, keyEnd.length * 2);
+ keyObjects = Arrays.copyOf(keyObjects, keyObjects.length * 2);
+ valueObjects = Arrays.copyOf(valueObjects, valueObjects.length * 2);
+ keyInited = Arrays.copyOf(keyInited, keyInited.length * 2);
+ valueInited = Arrays.copyOf(valueInited, valueInited.length * 2);
}
}
@@ -126,44 +122,44 @@
*/
private void parse() {
parsed = true;
-
+
byte itemSeparator = oi.getItemSeparator();
- byte keyValueSeparator = oi.getKeyValueSeparator();
+ byte keyValueSeparator = oi.getKeyValueSeparator();
boolean isEscaped = oi.isEscaped();
byte escapeChar = oi.getEscapeChar();
-
+
// empty array?
if (length == 0) {
mapSize = 0;
return;
}
-
+
mapSize = 0;
int arrayByteEnd = start + length;
int elementByteBegin = start;
int keyValueSeparatorPosition = -1;
int elementByteEnd = start;
byte[] bytes = this.bytes.getData();
-
+
// Go through all bytes in the byte[]
while (elementByteEnd <= arrayByteEnd) {
// End of entry reached?
- if (elementByteEnd == arrayByteEnd
+ if (elementByteEnd == arrayByteEnd
|| bytes[elementByteEnd] == itemSeparator) {
// Array full?
if (keyStart == null || mapSize + 1 == keyStart.length) {
enlargeArrays();
}
keyStart[mapSize] = elementByteBegin;
- // If no keyValueSeparator is seen, all bytes belong to key, and
+ // If no keyValueSeparator is seen, all bytes belong to key, and
// value will be NULL.
- keyEnd[mapSize] = (keyValueSeparatorPosition == -1
- ? elementByteEnd: keyValueSeparatorPosition);
+ keyEnd[mapSize] = (keyValueSeparatorPosition == -1 ? elementByteEnd
+ : keyValueSeparatorPosition);
// reset keyValueSeparatorPosition
keyValueSeparatorPosition = -1;
mapSize++;
elementByteBegin = elementByteEnd + 1;
- elementByteEnd ++;
+ elementByteEnd++;
} else {
// Is this the first keyValueSeparator in this entry?
if (keyValueSeparatorPosition == -1
@@ -171,15 +167,15 @@
keyValueSeparatorPosition = elementByteEnd;
}
if (isEscaped && bytes[elementByteEnd] == escapeChar
- && elementByteEnd+1 < arrayByteEnd) {
+ && elementByteEnd + 1 < arrayByteEnd) {
// ignore the char after escape_char
elementByteEnd += 2;
} else {
- elementByteEnd ++;
+ elementByteEnd++;
}
}
}
-
+
// This makes sure we can use the same formula to compute the
// length of each value in the map.
keyStart[mapSize] = arrayByteEnd + 1;
@@ -189,19 +185,20 @@
Arrays.fill(valueInited, 0, mapSize, false);
}
}
-
+
/**
* Get the value in the map for the key.
*
- * If there are multiple matches (which is possible in the serialized
- * format), only the first one is returned.
+ * If there are multiple matches (which is possible in the serialized format),
+ * only the first one is returned.
*
- * The most efficient way to get the value for the key is to serialize the
- * key and then try to find it in the array. We do linear search because in
- * most cases, user only wants to get one or two values out of the map, and
- * the cost of building up a HashMap is substantially higher.
+ * The most efficient way to get the value for the key is to serialize the key
+ * and then try to find it in the array. We do linear search because in most
+ * cases, user only wants to get one or two values out of the map, and the
+ * cost of building up a HashMap is substantially higher.
*
- * @param key The key object that we are looking for.
+ * @param key
+ * The key object that we are looking for.
* @return The corresponding value object, or NULL if not found
*/
public Object getMapValueElement(Object key) {
@@ -209,83 +206,94 @@
parse();
}
// search for the key
- for (int i=0; i<mapSize; i++) {
- LazyPrimitive<?,?> lazyKeyI = uncheckedGetKey(i);
- if (lazyKeyI == null) continue;
- // getWritableObject() will convert LazyPrimitive to actual primitive writable objects.
+ for (int i = 0; i < mapSize; i++) {
+ LazyPrimitive<?, ?> lazyKeyI = uncheckedGetKey(i);
+ if (lazyKeyI == null) {
+ continue;
+ }
+ // getWritableObject() will convert LazyPrimitive to actual primitive
+ // writable objects.
Object keyI = lazyKeyI.getWritableObject();
- if (keyI == null) continue;
+ if (keyI == null) {
+ continue;
+ }
if (keyI.equals(key)) {
// Got a match, return the value
LazyObject v = uncheckedGetValue(i);
return v == null ? v : v.getObject();
}
}
-
+
return null;
}
/**
* Get the value object with the index without checking parsed.
- * @param index The index into the array starting from 0
+ *
+ * @param index
+ * The index into the array starting from 0
*/
private LazyObject uncheckedGetValue(int index) {
Text nullSequence = oi.getNullSequence();
int valueIBegin = keyEnd[index] + 1;
- int valueILength = keyStart[index+1] - 1 - valueIBegin;
- if (valueILength < 0 ||
- ((valueILength == nullSequence.getLength())
- && 0 == LazyUtils.compare(bytes.getData(), valueIBegin, valueILength,
- nullSequence.getBytes(), 0, nullSequence.getLength()))) {
- return null;
+ int valueILength = keyStart[index + 1] - 1 - valueIBegin;
+ if (valueILength < 0
+ || ((valueILength == nullSequence.getLength()) && 0 == LazyUtils
+ .compare(bytes.getData(), valueIBegin, valueILength, nullSequence
+ .getBytes(), 0, nullSequence.getLength()))) {
+ return null;
}
if (!valueInited[index]) {
valueInited[index] = true;
if (valueObjects[index] == null) {
- valueObjects[index] = LazyFactory.createLazyObject(
- ((MapObjectInspector)oi).getMapValueObjectInspector());
+ valueObjects[index] = LazyFactory
+ .createLazyObject(((MapObjectInspector) oi)
+ .getMapValueObjectInspector());
}
valueObjects[index].init(bytes, valueIBegin, valueILength);
}
return valueObjects[index];
}
-
+
/**
* Get the key object with the index without checking parsed.
- * @param index The index into the array starting from 0
+ *
+ * @param index
+ * The index into the array starting from 0
*/
- private LazyPrimitive<?,?> uncheckedGetKey(int index) {
- Text nullSequence = oi.getNullSequence();
+ private LazyPrimitive<?, ?> uncheckedGetKey(int index) {
+ Text nullSequence = oi.getNullSequence();
int keyIBegin = keyStart[index];
int keyILength = keyEnd[index] - keyStart[index];
- if (keyILength < 0 ||
- ((keyILength == nullSequence.getLength())
- && 0 == LazyUtils.compare(bytes.getData(), keyIBegin, keyILength,
- nullSequence.getBytes(), 0, nullSequence.getLength()))) {
+ if (keyILength < 0
+ || ((keyILength == nullSequence.getLength()) && 0 == LazyUtils.compare(
+ bytes.getData(), keyIBegin, keyILength, nullSequence.getBytes(), 0,
+ nullSequence.getLength()))) {
return null;
}
if (!keyInited[index]) {
keyInited[index] = true;
if (keyObjects[index] == null) {
// Keys are always primitive
- keyObjects[index] = LazyFactory.createLazyPrimitiveClass(
- (PrimitiveObjectInspector)((MapObjectInspector)oi).getMapKeyObjectInspector());
+ keyObjects[index] = LazyFactory
+ .createLazyPrimitiveClass((PrimitiveObjectInspector) ((MapObjectInspector) oi)
+ .getMapKeyObjectInspector());
}
keyObjects[index].init(bytes, keyIBegin, keyILength);
}
return keyObjects[index];
}
-
+
/**
- * cachedMap is reused for different calls to getMap().
- * But each LazyMap has a separate cachedMap so we won't overwrite the
- * data by accident.
+ * cachedMap is reused for different calls to getMap(). But each LazyMap has a
+ * separate cachedMap so we won't overwrite the data by accident.
*/
LinkedHashMap<Object, Object> cachedMap;
-
+
/**
- * Return the map object representing this LazyMap.
- * Note that the keyObjects will be Writable primitive objects.
+ * Return the map object representing this LazyMap. Note that the keyObjects
+ * will be Writable primitive objects.
+ *
* @return the map object
*/
public Map<Object, Object> getMap() {
@@ -298,11 +306,13 @@
} else {
cachedMap.clear();
}
-
+
// go through each element of the map
for (int i = 0; i < mapSize; i++) {
- LazyPrimitive<?,?> lazyKey = uncheckedGetKey(i);
- if (lazyKey == null) continue;
+ LazyPrimitive<?, ?> lazyKey = uncheckedGetKey(i);
+ if (lazyKey == null) {
+ continue;
+ }
Object key = lazyKey.getObject();
// do not overwrite if there are duplicate keys
if (key != null && !cachedMap.containsKey(key)) {
@@ -316,7 +326,8 @@
/**
* Get the size of the map represented by this LazyMap.
- * @return The size of the map, -1 for NULL map.
+ *
+ * @return The size of the map, -1 for NULL map.
*/
public int getMapSize() {
if (!parsed) {
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyNonPrimitive.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyNonPrimitive.java?rev=901604&r1=901603&r2=901604&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyNonPrimitive.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyNonPrimitive.java Thu Jan 21 08:59:22 2010
@@ -18,12 +18,12 @@
package org.apache.hadoop.hive.serde2.lazy;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.io.WritableComparator;
/**
* LazyPrimitive stores a primitive Object in a LazyObject.
*/
-public abstract class LazyNonPrimitive<OI extends ObjectInspector> extends LazyObject<OI> {
+public abstract class LazyNonPrimitive<OI extends ObjectInspector> extends
+ LazyObject<OI> {
protected ByteArrayRef bytes;
protected int start;
@@ -31,12 +31,14 @@
/**
* Create a LazyNonPrimitive object with the specified ObjectInspector.
- * @param oi The ObjectInspector would have to have a hierarchy of
- * LazyObjectInspectors with the leaf nodes being
- * WritableObjectInspectors. It's used both for accessing the
- * type hierarchy of the complex object, as well as getting
- * meta information (separator, nullSequence, etc) when parsing
- * the lazy object.
+ *
+ * @param oi
+ * The ObjectInspector would have to have a hierarchy of
+ * LazyObjectInspectors with the leaf nodes being
+ * WritableObjectInspectors. It's used both for accessing the type
+ * hierarchy of the complex object, as well as getting meta
+ * information (separator, nullSequence, etc) when parsing the lazy
+ * object.
*/
protected LazyNonPrimitive(OI oi) {
super(oi);
@@ -44,7 +46,7 @@
start = 0;
length = 0;
}
-
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
if (bytes == null) {
@@ -59,9 +61,10 @@
@Override
public Object getObject() {
- return this;
+ return this;
}
-
+
+ @Override
public int hashCode() {
return LazyUtils.hashBytes(bytes.getData(), start, length);
}
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java?rev=901604&r1=901603&r2=901604&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java Thu Jan 21 08:59:22 2010
@@ -22,40 +22,45 @@
/**
* LazyObject stores an object in a range of bytes in a byte[].
*
- * A LazyObject can represent any primitive object or hierarchical object
- * like array, map or struct.
+ * A LazyObject can represent any primitive object or hierarchical object like
+ * array, map or struct.
*/
public abstract class LazyObject<OI extends ObjectInspector> {
OI oi;
-
+
/**
* Create a LazyObject.
- * @param oi Derived classes can access meta information about this Lazy
- * Object (e.g, separator, nullSequence, escaper) from it.
+ *
+ * @param oi
+ * Derived classes can access meta information about this Lazy Object
+ * (e.g, separator, nullSequence, escaper) from it.
*/
protected LazyObject(OI oi) {
this.oi = oi;
}
-
+
/**
- * Set the data for this LazyObject.
- * We take ByteArrayRef instead of byte[] so that we will be able to drop
- * the reference to byte[] by a single assignment.
- * The ByteArrayRef object can be reused across multiple rows.
- * @param bytes The wrapper of the byte[].
- * @param start The start position inside the bytes.
- * @param length The length of the data, starting from "start"
+ * Set the data for this LazyObject. We take ByteArrayRef instead of byte[] so
+ * that we will be able to drop the reference to byte[] by a single
+ * assignment. The ByteArrayRef object can be reused across multiple rows.
+ *
+ * @param bytes
+ * The wrapper of the byte[].
+ * @param start
+ * The start position inside the bytes.
+ * @param length
+ * The length of the data, starting from "start"
* @see ByteArrayRef
*/
public abstract void init(ByteArrayRef bytes, int start, int length);
/**
- * If the LazyObject is a primitive Object, then deserialize it and return
- * the actual primitive Object.
- * Otherwise (array, map, struct), return this.
+ * If the LazyObject is a primitive Object, then deserialize it and return the
+ * actual primitive Object. Otherwise (array, map, struct), return this.
*/
public abstract Object getObject();
-
+
+ @Override
public abstract int hashCode();
}
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java?rev=901604&r1=901603&r2=901604&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java Thu Jan 21 08:59:22 2010
@@ -23,8 +23,8 @@
/**
* LazyPrimitive stores a primitive Object in a LazyObject.
*/
-public abstract class LazyPrimitive<OI extends ObjectInspector,
- T extends Writable> extends LazyObject<OI> {
+public abstract class LazyPrimitive<OI extends ObjectInspector, T extends Writable>
+ extends LazyObject<OI> {
LazyPrimitive(OI oi) {
super(oi);
@@ -39,9 +39,10 @@
boolean isNull = false;
/**
- * Returns the primitive object represented by this LazyObject.
- * This is useful because it can make sure we have "null" for null objects.
+ * Returns the primitive object represented by this LazyObject. This is useful
+ * because it can make sure we have "null" for null objects.
*/
+ @Override
public Object getObject() {
return isNull ? null : this;
}
@@ -49,13 +50,15 @@
public T getWritableObject() {
return isNull ? null : data;
}
-
+
+ @Override
public String toString() {
return isNull ? null : data.toString();
}
-
- public int hashCode(){
+
+ @Override
+ public int hashCode() {
return isNull ? 0 : data.hashCode();
}
-
+
}
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java?rev=901604&r1=901603&r2=901604&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java Thu Jan 21 08:59:22 2010
@@ -33,18 +33,19 @@
* </p>
*
*/
-public class LazyShort extends LazyPrimitive<LazyShortObjectInspector, ShortWritable> {
+public class LazyShort extends
+ LazyPrimitive<LazyShortObjectInspector, ShortWritable> {
public LazyShort(LazyShortObjectInspector oi) {
super(oi);
data = new ShortWritable();
}
-
+
public LazyShort(LazyShort copy) {
super(copy);
data = new ShortWritable(copy.data.get());
}
-
+
@Override
public void init(ByteArrayRef bytes, int start, int length) {
try {
@@ -63,12 +64,12 @@
* @param bytes
* @param start
* @param length
- * a UTF-8 encoded string representation of a short quantity.
+ * a UTF-8 encoded string representation of a short quantity.
* @return short the value represented by the argument
* @exception NumberFormatException
- * if the argument could not be parsed as a short quantity.
+ * if the argument could not be parsed as a short quantity.
*/
- public static short parseShort(byte[] bytes, int start, int length)
+ public static short parseShort(byte[] bytes, int start, int length)
throws NumberFormatException {
return parseShort(bytes, start, length, 10);
}
@@ -76,18 +77,18 @@
/**
* Parses the string argument as if it was a short value and returns the
* result. Throws NumberFormatException if the string does not represent a
- * single short quantity. The second argument specifies the radix to use
- * when parsing the value.
+ * single short quantity. The second argument specifies the radix to use when
+ * parsing the value.
*
* @param bytes
* @param start
* @param length
- * a UTF-8 encoded string representation of a short quantity.
+ * a UTF-8 encoded string representation of a short quantity.
* @param radix
- * the radix to use when parsing.
+ * the radix to use when parsing.
* @return short the value represented by the argument
* @exception NumberFormatException
- * if the argument could not be parsed as a short quantity.
+ * if the argument could not be parsed as a short quantity.
*/
public static short parseShort(byte[] bytes, int start, int length, int radix)
throws NumberFormatException {