You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2012/03/09 01:47:38 UTC
svn commit: r1298673 [2/4] - in /hive/trunk:
hbase-handler/src/java/org/apache/hadoop/hive/hbase/
hbase-handler/src/test/org/apache/hadoop/hive/hbase/
hbase-handler/src/test/queries/ hbase-handler/src/test/results/
serde/src/java/org/apache/hadoop/hive...
Modified: hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java?rev=1298673&r1=1298672&r2=1298673&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java (original)
+++ hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java Fri Mar 9 00:47:37 2012
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.hbase;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import junit.framework.TestCase;
@@ -27,17 +28,28 @@ import junit.framework.TestCase;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hive.hbase.HBaseSerDe.ColumnMapping;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeUtils;
+import org.apache.hadoop.hive.serde2.io.ByteWritable;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
+import org.apache.hadoop.hive.serde2.lazy.LazyPrimitive;
import org.apache.hadoop.hive.serde2.lazy.LazyString;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
/**
* TestLazyHBaseObject is a test for the LazyHBaseXXX classes.
@@ -75,7 +87,11 @@ public class TestLazyHBaseObject extends
Result r = new Result(kvs);
- b.init(r, "cfb".getBytes());
+ List<Boolean> mapBinaryStorage = new ArrayList<Boolean>();
+ mapBinaryStorage.add(false);
+ mapBinaryStorage.add(false);
+
+ b.init(r, "cfb".getBytes(), mapBinaryStorage);
assertEquals(
new Text("def"),
@@ -131,7 +147,11 @@ public class TestLazyHBaseObject extends
Bytes.toBytes("cfc"), Bytes.toBytes("col3"), Bytes.toBytes("cfccol3")));
Result r = new Result(kvs);
- b.init(r, "cfb".getBytes());
+ List<Boolean> mapBinaryStorage = new ArrayList<Boolean>();
+ mapBinaryStorage.add(false);
+ mapBinaryStorage.add(false);
+
+ b.init(r, "cfb".getBytes(), mapBinaryStorage);
assertEquals(
new Text("d\tf"),
@@ -158,33 +178,305 @@ public class TestLazyHBaseObject extends
}
/**
+ * Test the LazyHBaseCellMap class for the case where both the key and the value in the family
+ * map are stored in binary format using the appropriate LazyPrimitive objects.
+ */
+ public void testLazyHBaseCellMap3() {
+
+ Text nullSequence = new Text("\\N");
+ TypeInfo mapBinaryIntKeyValue = TypeInfoUtils.getTypeInfoFromTypeString("map<int,int>");
+ ObjectInspector oi = LazyFactory.createLazyObjectInspector(
+ mapBinaryIntKeyValue, new byte [] {(byte)1, (byte) 2}, 0, nullSequence, false, (byte) 0);
+ LazyHBaseCellMap hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
+
+ List<KeyValue> kvs = new ArrayList<KeyValue>();
+ byte [] rowKey = "row-key".getBytes();
+ byte [] cfInt = "cf-int".getBytes();
+ kvs.add(new KeyValue(rowKey, cfInt, Bytes.toBytes(1), Bytes.toBytes(1)));
+ Result result = new Result(kvs);
+ List<Boolean> mapBinaryStorage = new ArrayList<Boolean>();
+ mapBinaryStorage.add(true);
+ mapBinaryStorage.add(true);
+ hbaseCellMap.init(result, cfInt, mapBinaryStorage);
+ IntWritable expectedIntValue = new IntWritable(1);
+ LazyPrimitive<?, ?> lazyPrimitive =
+ (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedIntValue);
+
+ assertEquals(expectedIntValue, lazyPrimitive.getWritableObject());
+
+ kvs.clear();
+ kvs.add(new KeyValue(
+ rowKey, cfInt, Bytes.toBytes(Integer.MIN_VALUE), Bytes.toBytes(Integer.MIN_VALUE)));
+ result = new Result(kvs);
+ hbaseCellMap.init(result, cfInt, mapBinaryStorage);
+ expectedIntValue = new IntWritable(Integer.MIN_VALUE);
+ lazyPrimitive =
+ (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedIntValue);
+
+ assertEquals(expectedIntValue, lazyPrimitive.getWritableObject());
+
+ kvs.clear();
+ kvs.add(new KeyValue(
+ rowKey, cfInt, Bytes.toBytes(Integer.MAX_VALUE), Bytes.toBytes(Integer.MAX_VALUE)));
+ result = new Result(kvs);
+ hbaseCellMap.init(result, cfInt, mapBinaryStorage);
+ expectedIntValue = new IntWritable(Integer.MAX_VALUE);
+ lazyPrimitive =
+ (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedIntValue);
+
+ assertEquals(expectedIntValue, lazyPrimitive.getWritableObject());
+
+ TypeInfo mapBinaryByteKeyValue =
+ TypeInfoUtils.getTypeInfoFromTypeString("map<tinyint,tinyint>");
+ oi = LazyFactory.createLazyObjectInspector(
+ mapBinaryByteKeyValue, new byte [] {(byte) 1, (byte) 2}, 0, nullSequence, false, (byte) 0);
+ hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
+ byte [] cfByte = "cf-byte".getBytes();
+ kvs.clear();
+ kvs.add(new KeyValue(rowKey, cfByte, new byte [] {(byte) 1}, new byte [] {(byte) 1}));
+ result = new Result(kvs);
+ hbaseCellMap.init(result, cfByte, mapBinaryStorage);
+ ByteWritable expectedByteValue = new ByteWritable((byte) 1);
+ lazyPrimitive =
+ (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedByteValue);
+
+ assertEquals(expectedByteValue, lazyPrimitive.getWritableObject());
+
+ kvs.clear();
+ kvs.add(new KeyValue(rowKey, cfByte, new byte [] {Byte.MIN_VALUE},
+ new byte [] {Byte.MIN_VALUE}));
+ result = new Result(kvs);
+ hbaseCellMap.init(result, cfByte, mapBinaryStorage);
+ expectedByteValue = new ByteWritable(Byte.MIN_VALUE);
+ lazyPrimitive =
+ (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedByteValue);
+
+ assertEquals(expectedByteValue, lazyPrimitive.getWritableObject());
+
+ kvs.clear();
+ kvs.add(new KeyValue(rowKey, cfByte, new byte [] {Byte.MAX_VALUE},
+ new byte [] {Byte.MAX_VALUE}));
+ result = new Result(kvs);
+ hbaseCellMap.init(result, cfByte, mapBinaryStorage);
+ expectedByteValue = new ByteWritable(Byte.MAX_VALUE);
+ lazyPrimitive =
+ (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedByteValue);
+
+ assertEquals(expectedByteValue, lazyPrimitive.getWritableObject());
+
+ TypeInfo mapBinaryShortKeyValue =
+ TypeInfoUtils.getTypeInfoFromTypeString("map<smallint,smallint>");
+ oi = LazyFactory.createLazyObjectInspector(
+ mapBinaryShortKeyValue, new byte [] {(byte) 1, (byte) 2}, 0, nullSequence, false, (byte) 0);
+ hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
+ byte [] cfShort = "cf-short".getBytes();
+ kvs.clear();
+ kvs.add(new KeyValue(rowKey, cfShort, Bytes.toBytes((short) 1), Bytes.toBytes((short) 1)));
+ result = new Result(kvs);
+ hbaseCellMap.init(result, cfShort, mapBinaryStorage);
+ ShortWritable expectedShortValue = new ShortWritable((short) 1);
+ lazyPrimitive =
+ (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedShortValue);
+
+ assertEquals(expectedShortValue, lazyPrimitive.getWritableObject());
+
+ kvs.clear();
+ kvs.add(new KeyValue(rowKey, cfShort, Bytes.toBytes(Short.MIN_VALUE),
+ Bytes.toBytes(Short.MIN_VALUE)));
+ result = new Result(kvs);
+ hbaseCellMap.init(result, cfShort, mapBinaryStorage);
+ expectedShortValue = new ShortWritable(Short.MIN_VALUE);
+ lazyPrimitive =
+ (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedShortValue);
+
+ assertEquals(expectedShortValue, lazyPrimitive.getWritableObject());
+
+ kvs.clear();
+ kvs.add(new KeyValue(rowKey, cfShort, Bytes.toBytes(Short.MAX_VALUE),
+ Bytes.toBytes(Short.MAX_VALUE)));
+ result = new Result(kvs);
+ hbaseCellMap.init(result, cfShort, mapBinaryStorage);
+ expectedShortValue = new ShortWritable(Short.MAX_VALUE);
+ lazyPrimitive =
+ (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedShortValue);
+
+ assertEquals(expectedShortValue, lazyPrimitive.getWritableObject());
+
+ TypeInfo mapBinaryLongKeyValue =
+ TypeInfoUtils.getTypeInfoFromTypeString("map<bigint,bigint>");
+ oi = LazyFactory.createLazyObjectInspector(
+ mapBinaryLongKeyValue, new byte [] {(byte) 1, (byte) 2}, 0, nullSequence, false, (byte) 0);
+ hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
+ byte [] cfLong = "cf-long".getBytes();
+ kvs.clear();
+ kvs.add(new KeyValue(rowKey, cfLong, Bytes.toBytes((long) 1), Bytes.toBytes((long) 1)));
+ result = new Result(kvs);
+ hbaseCellMap.init(result, cfLong, mapBinaryStorage);
+ LongWritable expectedLongValue = new LongWritable(1);
+ lazyPrimitive =
+ (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedLongValue);
+
+ assertEquals(expectedLongValue, lazyPrimitive.getWritableObject());
+
+ kvs.clear();
+ kvs.add(new KeyValue(rowKey, cfLong, Bytes.toBytes(Long.MIN_VALUE),
+ Bytes.toBytes(Long.MIN_VALUE)));
+ result = new Result(kvs);
+ hbaseCellMap.init(result, cfLong, mapBinaryStorage);
+ expectedLongValue = new LongWritable(Long.MIN_VALUE);
+ lazyPrimitive =
+ (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedLongValue);
+
+ assertEquals(expectedLongValue, lazyPrimitive.getWritableObject());
+
+ kvs.clear();
+ kvs.add(new KeyValue(rowKey, cfLong, Bytes.toBytes(Long.MAX_VALUE),
+ Bytes.toBytes(Long.MAX_VALUE)));
+ result = new Result(kvs);
+ hbaseCellMap.init(result, cfLong, mapBinaryStorage);
+ expectedLongValue = new LongWritable(Long.MAX_VALUE);
+ lazyPrimitive =
+ (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedLongValue);
+
+ assertEquals(expectedLongValue, lazyPrimitive.getWritableObject());
+
+ TypeInfo mapBinaryFloatKeyValue =
+ TypeInfoUtils.getTypeInfoFromTypeString("map<float,float>");
+ oi = LazyFactory.createLazyObjectInspector(
+ mapBinaryFloatKeyValue, new byte [] {(byte) 1, (byte) 2}, 0, nullSequence, false,
+ (byte) 0);
+ hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
+ byte [] cfFloat = "cf-float".getBytes();
+ kvs.clear();
+ kvs.add(new KeyValue(rowKey, cfFloat, Bytes.toBytes((float) 1.0F),
+ Bytes.toBytes((float) 1.0F)));
+ result = new Result(kvs);
+ hbaseCellMap.init(result, cfFloat, mapBinaryStorage);
+ FloatWritable expectedFloatValue = new FloatWritable(1.0F);
+ lazyPrimitive =
+ (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedFloatValue);
+
+ assertEquals(expectedFloatValue, lazyPrimitive.getWritableObject());
+
+ kvs.clear();
+ kvs.add(new KeyValue(rowKey, cfFloat, Bytes.toBytes((float) Float.MIN_VALUE),
+ Bytes.toBytes((float) Float.MIN_VALUE)));
+ result = new Result(kvs);
+ hbaseCellMap.init(result, cfFloat, mapBinaryStorage);
+ expectedFloatValue = new FloatWritable(Float.MIN_VALUE);
+ lazyPrimitive =
+ (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedFloatValue);
+
+ assertEquals(expectedFloatValue, lazyPrimitive.getWritableObject());
+
+ kvs.clear();
+ kvs.add(new KeyValue(rowKey, cfFloat, Bytes.toBytes((float) Float.MAX_VALUE),
+ Bytes.toBytes((float) Float.MAX_VALUE)));
+ result = new Result(kvs);
+ hbaseCellMap.init(result, cfFloat, mapBinaryStorage);
+ expectedFloatValue = new FloatWritable(Float.MAX_VALUE);
+ lazyPrimitive =
+ (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedFloatValue);
+
+ assertEquals(expectedFloatValue, lazyPrimitive.getWritableObject());
+
+ TypeInfo mapBinaryDoubleKeyValue =
+ TypeInfoUtils.getTypeInfoFromTypeString("map<double,double>");
+ oi = LazyFactory.createLazyObjectInspector(
+ mapBinaryDoubleKeyValue, new byte [] {(byte) 1, (byte) 2}, 0, nullSequence, false,
+ (byte) 0);
+ hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
+ byte [] cfDouble = "cf-double".getBytes();
+ kvs.clear();
+ kvs.add(new KeyValue(rowKey, cfDouble, Bytes.toBytes(1.0), Bytes.toBytes(1.0)));
+ result = new Result(kvs);
+ hbaseCellMap.init(result, cfDouble, mapBinaryStorage);
+ DoubleWritable expectedDoubleValue = new DoubleWritable(1.0);
+ lazyPrimitive =
+ (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedDoubleValue);
+
+ assertEquals(expectedDoubleValue, lazyPrimitive.getWritableObject());
+
+ kvs.clear();
+ kvs.add(new KeyValue(rowKey, cfDouble, Bytes.toBytes(Double.MIN_VALUE),
+ Bytes.toBytes(Double.MIN_VALUE)));
+ result = new Result(kvs);
+ hbaseCellMap.init(result, cfDouble, mapBinaryStorage);
+ expectedDoubleValue = new DoubleWritable(Double.MIN_VALUE);
+ lazyPrimitive =
+ (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedDoubleValue);
+
+ assertEquals(expectedDoubleValue, lazyPrimitive.getWritableObject());
+
+ kvs.clear();
+ kvs.add(new KeyValue(rowKey, cfDouble, Bytes.toBytes(Double.MAX_VALUE),
+ Bytes.toBytes(Double.MAX_VALUE)));
+ result = new Result(kvs);
+ hbaseCellMap.init(result, cfDouble, mapBinaryStorage);
+ expectedDoubleValue = new DoubleWritable(Double.MAX_VALUE);
+ lazyPrimitive =
+ (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedDoubleValue);
+
+ assertEquals(expectedDoubleValue, lazyPrimitive.getWritableObject());
+
+ TypeInfo mapBinaryBooleanKeyValue =
+ TypeInfoUtils.getTypeInfoFromTypeString("map<boolean,boolean>");
+ oi = LazyFactory.createLazyObjectInspector(
+ mapBinaryBooleanKeyValue, new byte [] {(byte) 1, (byte) 2}, 0, nullSequence, false,
+ (byte) 0);
+ hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
+ byte [] cfBoolean = "cf-boolean".getBytes();
+ kvs.clear();
+ kvs.add(new KeyValue(rowKey, cfBoolean, Bytes.toBytes(false), Bytes.toBytes(false)));
+ result = new Result(kvs);
+ hbaseCellMap.init(result, cfBoolean, mapBinaryStorage);
+ BooleanWritable expectedBooleanValue = new BooleanWritable(false);
+ lazyPrimitive =
+ (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedBooleanValue);
+
+ assertEquals(expectedBooleanValue, lazyPrimitive.getWritableObject());
+
+ kvs.clear();
+ kvs.add(new KeyValue(rowKey, cfBoolean, Bytes.toBytes(true), Bytes.toBytes(true)));
+ result = new Result(kvs);
+ hbaseCellMap.init(result, cfBoolean, mapBinaryStorage);
+ expectedBooleanValue = new BooleanWritable(true);
+ lazyPrimitive =
+ (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedBooleanValue);
+
+ assertEquals(expectedBooleanValue, lazyPrimitive.getWritableObject());
+ }
+
+ /**
* Test the LazyHBaseRow class with one-for-one mappings between
* Hive fields and HBase columns.
*/
public void testLazyHBaseRow1() {
List<TypeInfo> fieldTypeInfos =
TypeInfoUtils.getTypeInfosFromTypeString(
- "string,int,array<string>,map<string,string>,string");
- List<String> fieldNames = Arrays.asList(
- new String[]{"key", "a", "b", "c", "d"});
+ "string,int,array<string>,map<string,string>,string");
+ List<String> fieldNames = Arrays.asList(new String[]{"key", "a", "b", "c", "d"});
Text nullSequence = new Text("\\N");
String hbaseColsMapping = ":key,cfa:a,cfa:b,cfb:c,cfb:d";
- List<String> colFamily = new ArrayList<String>();
- List<String> colQual = new ArrayList<String>();
- List<byte []> colFamilyBytes = new ArrayList<byte []>();
- List<byte []> colQualBytes = new ArrayList<byte []>();
-
- int iKey = -1;
+ List<ColumnMapping> columnsMapping = null;
try {
- iKey = HBaseSerDe.parseColumnMapping(
- hbaseColsMapping, colFamily, colFamilyBytes, colQual, colQualBytes);
+ columnsMapping = HBaseSerDe.parseColumnsMapping(hbaseColsMapping);
} catch (SerDeException e) {
fail(e.toString());
}
- assertEquals(0, iKey);
+ for (int i = 0; i < columnsMapping.size(); i++) {
+ ColumnMapping colMap = columnsMapping.get(i);
+
+ if (!colMap.hbaseRowKey && colMap.qualifierName == null) {
+ colMap.binaryStorage.add(false);
+ colMap.binaryStorage.add(false);
+ } else {
+ colMap.binaryStorage.add(false);
+ }
+ }
ObjectInspector oi = LazyFactory.createLazyStructInspector(fieldNames,
fieldTypeInfos, new byte[] {' ', ':', '='},
@@ -203,7 +495,7 @@ public class TestLazyHBaseObject extends
Bytes.toBytes("cfb"), Bytes.toBytes("d"), Bytes.toBytes("hi")));
Result r = new Result(kvs);
- o.init(r, colFamily, colFamilyBytes, colQual, colQualBytes);
+ o.init(r, columnsMapping);
assertEquals(
("{'key':'test-row','a':123,'b':['a','b','c'],"
@@ -217,7 +509,7 @@ public class TestLazyHBaseObject extends
Bytes.toBytes("cfb"), Bytes.toBytes("c"), Bytes.toBytes("d=e:f=g")));
r = new Result(kvs);
- o.init(r, colFamily, colFamilyBytes, colQual, colQualBytes);
+ o.init(r, columnsMapping);
assertEquals(
("{'key':'test-row','a':123,'b':null,"
@@ -233,7 +525,7 @@ public class TestLazyHBaseObject extends
Bytes.toBytes("cfb"), Bytes.toBytes("d"), Bytes.toBytes("no")));
r = new Result(kvs);
- o.init(r, colFamily, colFamilyBytes, colQual, colQualBytes);
+ o.init(r, columnsMapping);
assertEquals(
("{'key':'test-row','a':null,'b':['a'],"
@@ -247,7 +539,7 @@ public class TestLazyHBaseObject extends
Bytes.toBytes("cfb"), Bytes.toBytes("d"), Bytes.toBytes("no")));
r = new Result(kvs);
- o.init(r, colFamily, colFamilyBytes, colQual, colQualBytes);
+ o.init(r, columnsMapping);
assertEquals(
("{'key':'test-row','a':null,'b':['','a','',''],"
@@ -265,7 +557,7 @@ public class TestLazyHBaseObject extends
Bytes.toBytes("cfb"), Bytes.toBytes("d"), Bytes.toBytes("")));
r = new Result(kvs);
- o.init(r, colFamily, colFamilyBytes, colQual, colQualBytes);
+ o.init(r, columnsMapping);
assertEquals(
"{'key':'test-row','a':123,'b':[],'c':{},'d':''}".replace("'", "\""),
@@ -284,20 +576,25 @@ public class TestLazyHBaseObject extends
List<String> fieldNames = Arrays.asList(
new String[]{"key", "a", "b", "c", "d"});
Text nullSequence = new Text("\\N");
-
+ List<ColumnMapping> columnsMapping = null;
String hbaseColsMapping = ":key,cfa:a,cfa:b,cfb:,cfc:d";
- List<String> colFamily = new ArrayList<String>();
- List<String> colQual = new ArrayList<String>();
- List<byte []> colFamilyBytes = new ArrayList<byte []>();
- List<byte []> colQualBytes = new ArrayList<byte []>();
- int iKey = -1;
+
try {
- iKey = HBaseSerDe.parseColumnMapping(
- hbaseColsMapping, colFamily, colFamilyBytes, colQual, colQualBytes);
+ columnsMapping = HBaseSerDe.parseColumnsMapping(hbaseColsMapping);
} catch (SerDeException e) {
fail(e.toString());
}
- assertEquals(0, iKey);
+
+ for (int i = 0; i < columnsMapping.size(); i++) {
+ ColumnMapping colMap = columnsMapping.get(i);
+
+ if (!colMap.hbaseRowKey && colMap.qualifierName == null) {
+ colMap.binaryStorage.add(false);
+ colMap.binaryStorage.add(false);
+ } else {
+ colMap.binaryStorage.add(false);
+ }
+ }
ObjectInspector oi = LazyFactory.createLazyStructInspector(
fieldNames,
@@ -319,7 +616,7 @@ public class TestLazyHBaseObject extends
Bytes.toBytes("cfc"), Bytes.toBytes("d"), Bytes.toBytes("hi")));
Result r = new Result(kvs);
- o.init(r, colFamily, colFamilyBytes, colQual, colQualBytes);
+ o.init(r, columnsMapping);
assertEquals(
("{'key':'test-row','a':123,'b':['a','b','c'],"
@@ -335,7 +632,7 @@ public class TestLazyHBaseObject extends
Bytes.toBytes("cfb"), Bytes.toBytes("f"), Bytes.toBytes("g")));
r = new Result(kvs);
- o.init(r, colFamily, colFamilyBytes, colQual, colQualBytes);
+ o.init(r, columnsMapping);
assertEquals(
("{'key':'test-row','a':123,'b':null,"
@@ -351,7 +648,7 @@ public class TestLazyHBaseObject extends
Bytes.toBytes("cfc"), Bytes.toBytes("d"), Bytes.toBytes("no")));
r = new Result(kvs);
- o.init(r, colFamily, colFamilyBytes, colQual, colQualBytes);
+ o.init(r, columnsMapping);
assertEquals(
("{'key':'test-row','a':null,'b':['a'],"
@@ -365,7 +662,7 @@ public class TestLazyHBaseObject extends
Bytes.toBytes("cfc"), Bytes.toBytes("d"), Bytes.toBytes("no")));
r = new Result(kvs);
- o.init(r, colFamily, colFamilyBytes, colQual, colQualBytes);
+ o.init(r, columnsMapping);
assertEquals(
("{'key':'test-row','a':null,'b':['','a','',''],"
@@ -381,10 +678,164 @@ public class TestLazyHBaseObject extends
Bytes.toBytes("cfc"), Bytes.toBytes("d"), Bytes.toBytes("")));
r = new Result(kvs);
- o.init(r, colFamily, colFamilyBytes, colQual, colQualBytes);
+ o.init(r, columnsMapping);
assertEquals(
"{'key':'test-row','a':123,'b':[],'c':{},'d':''}".replace("'", "\""),
SerDeUtils.getJSONString(o, oi));
}
+
+ /**
+ * Test the LazyHBaseRow class with a one-to-one/onto mapping between Hive columns and
+ * HBase column family/column qualifier pairs. The column types are primitive and fields
+ * are stored in binary format in HBase.
+ */
+ public void testLazyHBaseRow3() {
+
+ List<TypeInfo> fieldTypeInfos = TypeInfoUtils.getTypeInfosFromTypeString(
+ "string,int,tinyint,smallint,bigint,float,double,string,boolean");
+ List<String> fieldNames = Arrays.asList(
+ new String [] {"key", "c_int", "c_byte", "c_short", "c_long", "c_float", "c_double",
+ "c_string", "c_bool"});
+ Text nullSequence = new Text("\\N");
+ String hbaseColumnsMapping = ":key#str,cf-int:cq-int#bin,cf-byte:cq-byte#bin,"
+ + "cf-short:cq-short#bin,cf-long:cq-long#bin,cf-float:cq-float#bin,cf-double:cq-double#bin,"
+ + "cf-string:cq-string#str,cf-bool:cq-bool#bin";
+ List<ColumnMapping> columnsMapping = null;
+
+ try {
+ columnsMapping = HBaseSerDe.parseColumnsMapping(hbaseColumnsMapping);
+ } catch (SerDeException sde) {
+ fail(sde.toString());
+ }
+
+ for (int i = 0; i < columnsMapping.size(); i++) {
+ ColumnMapping colMap = columnsMapping.get(i);
+
+ if (i == 0 || i == 7) {
+ colMap.binaryStorage.add(false);
+ } else {
+ colMap.binaryStorage.add(true);
+ }
+ }
+
+ ObjectInspector oi =
+ LazyFactory.createLazyStructInspector(fieldNames, fieldTypeInfos,
+ new byte [] {' ', ':', '='}, nullSequence, false, false, (byte) 0);
+
+ LazyHBaseRow o = new LazyHBaseRow((LazySimpleStructObjectInspector) oi);
+
+ byte [] rowKey = "row-key".getBytes();
+ List<KeyValue> kvs = new ArrayList<KeyValue>();
+ byte [] value;
+
+ for (int i = 1; i < columnsMapping.size(); i++) {
+
+ switch (i) {
+
+ case 1:
+ value = Bytes.toBytes(1);
+ break;
+
+ case 2:
+ value = new byte[]{(byte)1};
+ break;
+
+ case 3:
+ value = Bytes.toBytes((short) 1);
+ break;
+
+ case 4:
+ value = Bytes.toBytes((long) 1);
+ break;
+
+ case 5:
+ value = Bytes.toBytes((float) 1.0F);
+ break;
+
+ case 6:
+ value = Bytes.toBytes((double) 1.0);
+ break;
+
+ case 7:
+ value = "Hadoop, Hive, with HBase storage handler.".getBytes();
+ break;
+
+ case 8:
+ value = Bytes.toBytes(true);
+ break;
+
+ default:
+ throw new RuntimeException("Not expected: " + i);
+ }
+
+ ColumnMapping colMap = columnsMapping.get(i);
+ kvs.add(new KeyValue(rowKey, colMap.familyNameBytes, colMap.qualifierNameBytes, value));
+ }
+
+ Collections.sort(kvs, KeyValue.COMPARATOR);
+ Result result = new Result(kvs);
+ o.init(result, columnsMapping);
+ List<? extends StructField> fieldRefs = ((StructObjectInspector) oi).getAllStructFieldRefs();
+
+
+ for (int i = 0; i < fieldRefs.size(); i++) {
+ Object fieldData = ((StructObjectInspector) oi).getStructFieldData(o, fieldRefs.get(i));
+
+ assert(fieldData != null);
+ assert(fieldData instanceof LazyPrimitive<?, ?>);
+ Writable writable = ((LazyPrimitive<?, ?>) fieldData).getWritableObject();
+
+ switch (i) {
+ case 0:
+ Text text = new Text("row-key");
+ assertEquals(text, writable);
+ break;
+
+ case 1:
+ IntWritable iw = new IntWritable(1);
+ assertEquals(iw, writable);
+ break;
+
+ case 2:
+ ByteWritable bw = new ByteWritable((byte) 1);
+ assertEquals(bw, writable);
+ break;
+
+ case 3:
+ ShortWritable sw = new ShortWritable((short) 1);
+ assertEquals(sw, writable);
+ break;
+
+ case 4:
+ LongWritable lw = new LongWritable(1);
+ assertEquals(lw, writable);
+ break;
+
+ case 5:
+ FloatWritable fw = new FloatWritable(1.0F);
+ assertEquals(fw, writable);
+ break;
+
+ case 6:
+ DoubleWritable dw = new DoubleWritable(1.0);
+ assertEquals(dw, writable);
+ break;
+
+ case 7:
+ Text t = new Text("Hadoop, Hive, with HBase storage handler.");
+ assertEquals(t, writable);
+ break;
+
+ case 8:
+ BooleanWritable boolWritable = new BooleanWritable(true);
+ assertEquals(boolWritable, writable);
+ break;
+
+ default:
+ fail("Error: Unanticipated value in deserializing fields for HBaseSerDe.");
+ break;
+ }
+ }
+ }
}
Added: hive/trunk/hbase-handler/src/test/queries/hbase_binary_external_table_queries.q
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/queries/hbase_binary_external_table_queries.q?rev=1298673&view=auto
==============================================================================
--- hive/trunk/hbase-handler/src/test/queries/hbase_binary_external_table_queries.q (added)
+++ hive/trunk/hbase-handler/src/test/queries/hbase_binary_external_table_queries.q Fri Mar 9 00:47:37 2012
@@ -0,0 +1,38 @@
+DROP TABLE t_ext_hbase_1;
+
+CREATE EXTERNAL TABLE t_ext_hbase_1
+(key STRING, c_bool BOOLEAN, c_byte TINYINT, c_short SMALLINT,
+ c_int INT, c_long BIGINT, c_string STRING, c_float FLOAT, c_double DOUBLE)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:cq-boolean,cf:cq-byte,cf:cq-short,cf:cq-int,cf:cq-long,cf:cq-string,cf:cq-float,cf:cq-double")
+TBLPROPERTIES ("hbase.table.name" = "HiveExternalTable");
+
+SELECT * FROM t_ext_hbase_1;
+
+DROP TABLE t_ext_hbase_1;
+DROP TABLE t_ext_hbase_2;
+
+CREATE EXTERNAL TABLE t_ext_hbase_2
+(key STRING, c_bool BOOLEAN, c_byte TINYINT, c_short SMALLINT,
+ c_int INT, c_long BIGINT, c_string STRING, c_float FLOAT, c_double DOUBLE)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key#b,cf:cq-boolean#b,cf:cq-byte#b,cf:cq-short#b,cf:cq-int#b,cf:cq-long#b,cf:cq-string#b,cf:cq-float#b,cf:cq-double#b")
+TBLPROPERTIES ("hbase.table.name" = "HiveExternalTable");
+
+SELECT * FROM t_ext_hbase_2;
+
+DROP TABLE t_ext_hbase_2;
+DROP TABLE t_ext_hbase_3;
+
+CREATE EXTERNAL TABLE t_ext_hbase_3
+(key STRING, c_bool BOOLEAN, c_byte TINYINT, c_short SMALLINT,
+ c_int INT, c_long BIGINT, c_string STRING, c_float FLOAT, c_double DOUBLE)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:cq-boolean,cf:cq-byte,cf:cq-short,cf:cq-int,cf:cq-long,cf:cq-string,cf:cq-float,cf:cq-double")
+TBLPROPERTIES (
+"hbase.table.name" = "HiveExternalTable",
+"hbase.table.default.storage.type" = "binary");
+
+SELECT * from t_ext_hbase_3;
+
+DROP table t_ext_hbase_3;
Added: hive/trunk/hbase-handler/src/test/queries/hbase_binary_map_queries.q
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/queries/hbase_binary_map_queries.q?rev=1298673&view=auto
==============================================================================
--- hive/trunk/hbase-handler/src/test/queries/hbase_binary_map_queries.q (added)
+++ hive/trunk/hbase-handler/src/test/queries/hbase_binary_map_queries.q Fri Mar 9 00:47:37 2012
@@ -0,0 +1,225 @@
+DROP TABLE hbase_src;
+
+CREATE TABLE hbase_src(key STRING,
+ tinyint_col TINYINT,
+ smallint_col SMALLINT,
+ int_col INT,
+ bigint_col BIGINT,
+ float_col FLOAT,
+ double_col DOUBLE,
+ string_col STRING);
+
+INSERT OVERWRITE TABLE hbase_src
+ SELECT key, key, key, key, key, key, key, value
+ FROM src
+ WHERE key = 125 OR key = 126 OR key = 127;
+
+DROP TABLE t_hbase_maps;
+
+CREATE TABLE t_hbase_maps(key STRING,
+ tinyint_map_col MAP<TINYINT, TINYINT>,
+ smallint_map_col MAP<SMALLINT, SMALLINT>,
+ int_map_col MAP<INT, INT>,
+ bigint_map_col MAP<BIGINT, BIGINT>,
+ float_map_col MAP<FLOAT, FLOAT>,
+ double_map_col MAP<DOUBLE, DOUBLE>,
+ string_map_col MAP<STRING, STRING>,
+ boolean_map_col MAP<BOOLEAN, BOOLEAN>)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping"=":key,cf-tinyint:,cf-smallint:,cf-int:,cf-bigint:,cf-float:,cf-double:,cf-string:,cf-boolean:")
+TBLPROPERTIES ("hbase.table.name"="t_hive_maps");
+
+INSERT OVERWRITE TABLE t_hbase_maps
+ SELECT key,
+ map(tinyint_col, tinyint_col),
+ map(smallint_col, smallint_col),
+ map(int_col, int_col),
+ map(bigint_col, bigint_col),
+ map(float_col, float_col),
+ map(double_col, double_col),
+ map(key, string_col),
+ map(true, true)
+ FROM hbase_src
+ WHERE key = 125;
+
+INSERT OVERWRITE TABLE t_hbase_maps
+ SELECT key,
+ map(tinyint_col, tinyint_col),
+ map(smallint_col, smallint_col),
+ map(int_col, int_col),
+ map(bigint_col, bigint_col),
+ map(float_col, float_col),
+ map(double_col, double_col),
+ map(key, string_col),
+ map(false, false)
+ FROM hbase_src
+ WHERE key = 126;
+
+SELECT * FROM t_hbase_maps ORDER BY key;
+
+DROP TABLE t_ext_hbase_maps;
+
+CREATE EXTERNAL TABLE t_ext_hbase_maps(key STRING,
+ tinyint_map_col MAP<TINYINT, TINYINT>,
+ smallint_map_col MAP<SMALLINT, SMALLINT>,
+ int_map_col MAP<INT, INT>,
+ bigint_map_col MAP<BIGINT, BIGINT>,
+ float_map_col MAP<FLOAT, FLOAT>,
+ double_map_col MAP<DOUBLE, DOUBLE>,
+ string_map_col MAP<STRING, STRING>,
+ boolean_map_col MAP<BOOLEAN, BOOLEAN>)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping"=":key,cf-tinyint:,cf-smallint:,cf-int:,cf-bigint:,cf-float:,cf-double:,cf-string:,cf-boolean:")
+TBLPROPERTIES ("hbase.table.name"="t_hive_maps");
+
+SELECT * FROM t_ext_hbase_maps ORDER BY key;
+
+DROP TABLE t_ext_hbase_maps;
+
+DROP TABLE t_ext_hbase_maps_1;
+
+CREATE EXTERNAL TABLE t_ext_hbase_maps_1(key STRING,
+ tinyint_map_col MAP<TINYINT, TINYINT>,
+ smallint_map_col MAP<SMALLINT, SMALLINT>,
+ int_map_col MAP<INT, INT>,
+ bigint_map_col MAP<BIGINT, BIGINT>,
+ float_map_col MAP<FLOAT, FLOAT>,
+ double_map_col MAP<DOUBLE, DOUBLE>,
+ string_map_col MAP<STRING, STRING>,
+ boolean_map_col MAP<BOOLEAN, BOOLEAN>)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping"=":key#b,cf-tinyint:#bi:bi,cf-smallint:#bin:bin,cf-int:#bina:bina,cf-bigint:#binar:binar,cf-float:#binary:binary,cf-double:#b:b,cf-string:#bi:bi,cf-boolean:#bin:bin")
+TBLPROPERTIES ("hbase.table.name"="t_hive_maps");
+
+SELECT * FROM t_ext_hbase_maps_1 ORDER BY key;
+
+DROP TABLE t_ext_hbase_maps_1;
+
+DROP TABLE t_ext_hbase_maps_2;
+
+CREATE EXTERNAL TABLE t_ext_hbase_maps_2(key STRING,
+ tinyint_map_col MAP<TINYINT, TINYINT>,
+ smallint_map_col MAP<SMALLINT, SMALLINT>,
+ int_map_col MAP<INT, INT>,
+ bigint_map_col MAP<BIGINT, BIGINT>,
+ float_map_col MAP<FLOAT, FLOAT>,
+ double_map_col MAP<DOUBLE, DOUBLE>,
+ string_map_col MAP<STRING, STRING>,
+ boolean_map_col MAP<BOOLEAN, BOOLEAN>)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping"=":key,cf-tinyint:,cf-smallint:,cf-int:,cf-bigint:,cf-float:,cf-double:,cf-string:,cf-boolean:")
+TBLPROPERTIES (
+"hbase.table.name"="t_hive_maps",
+"hbase.table.default.storage.type"="binary");
+
+SELECT * FROM t_ext_hbase_maps_2 ORDER BY key;
+
+DROP TABLE t_ext_hbase_maps_2;
+
+DROP TABLE t_hbase_maps_1;
+
+CREATE TABLE t_hbase_maps_1(key STRING,
+ tinyint_map_col MAP<TINYINT, TINYINT>,
+ smallint_map_col MAP<SMALLINT, SMALLINT>,
+ int_map_col MAP<INT, INT>,
+ bigint_map_col MAP<BIGINT, BIGINT>,
+ float_map_col MAP<FLOAT, FLOAT>,
+ double_map_col MAP<DOUBLE, DOUBLE>,
+ string_map_col MAP<STRING, STRING>,
+ boolean_map_col MAP<BOOLEAN, BOOLEAN>)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping"=":key#b,cf-tinyint:#b:b,cf-smallint:#b:b,cf-int:#b:b,cf-bigint:#b:b,cf-float:#b:b,cf-double:#b:b,cf-string:#b:b,cf-boolean:#b:b")
+TBLPROPERTIES ("hbase.table.name"="t_hive_maps_1");
+
+INSERT OVERWRITE TABLE t_hbase_maps_1
+ SELECT key,
+ map(tinyint_col, tinyint_col),
+ map(smallint_col, smallint_col),
+ map(int_col, int_col),
+ map(bigint_col, bigint_col),
+ map(float_col, float_col),
+ map(double_col, double_col),
+ map(key, string_col),
+ map(true, true)
+ FROM hbase_src
+ WHERE key = 125;
+
+INSERT OVERWRITE TABLE t_hbase_maps_1
+ SELECT key,
+ map(tinyint_col, tinyint_col),
+ map(smallint_col, smallint_col),
+ map(int_col, int_col),
+ map(bigint_col, bigint_col),
+ map(float_col, float_col),
+ map(double_col, double_col),
+ map(key, string_col),
+ map(false, false)
+ FROM hbase_src
+ WHERE key = 126;
+
+SELECT * FROM t_hbase_maps_1 ORDER BY key;
+
+DROP TABLE t_ext_hbase_maps_3;
+
+CREATE EXTERNAL TABLE t_ext_hbase_maps_3(key STRING,
+ tinyint_map_col MAP<TINYINT, TINYINT>,
+ smallint_map_col MAP<SMALLINT, SMALLINT>,
+ int_map_col MAP<INT, INT>,
+ bigint_map_col MAP<BIGINT, BIGINT>,
+ float_map_col MAP<FLOAT, FLOAT>,
+ double_map_col MAP<DOUBLE, DOUBLE>,
+ string_map_col MAP<STRING, STRING>,
+ boolean_map_col MAP<BOOLEAN, BOOLEAN>)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping"=":key#b,cf-tinyint:#bi:bi,cf-smallint:#bin:bin,cf-int:#bina:bina,cf-bigint:#binar:binar,cf-float:#binary:binary,cf-double:#b:b,cf-string:#bi:bi,cf-boolean:#bin:bin")
+TBLPROPERTIES ("hbase.table.name"="t_hive_maps_1");
+
+SELECT * FROM t_ext_hbase_maps_3 ORDER BY key;
+
+DROP TABLE t_ext_hbase_maps_3;
+
+DROP TABLE t_ext_hbase_maps_4;
+
+CREATE EXTERNAL TABLE t_ext_hbase_maps_4(key STRING,
+ tinyint_map_col MAP<TINYINT, TINYINT>,
+ smallint_map_col MAP<SMALLINT, SMALLINT>,
+ int_map_col MAP<INT, INT>,
+ bigint_map_col MAP<BIGINT, BIGINT>,
+ float_map_col MAP<FLOAT, FLOAT>,
+ double_map_col MAP<DOUBLE, DOUBLE>,
+ string_map_col MAP<STRING, STRING>,
+ boolean_map_col MAP<BOOLEAN, BOOLEAN>)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping"=":key,cf-tinyint:,cf-smallint:,cf-int:,cf-bigint:,cf-float:,cf-double:,cf-string:,cf-boolean:")
+TBLPROPERTIES ("hbase.table.name"="t_hive_maps_1");
+
+SELECT * FROM t_ext_hbase_maps_4 ORDER BY key;
+
+DROP TABLE t_ext_hbase_maps_4;
+
+DROP TABLE t_ext_hbase_maps_5;
+
+CREATE EXTERNAL TABLE t_ext_hbase_maps_5(key STRING,
+ tinyint_map_col MAP<TINYINT, TINYINT>,
+ smallint_map_col MAP<SMALLINT, SMALLINT>,
+ int_map_col MAP<INT, INT>,
+ bigint_map_col MAP<BIGINT, BIGINT>,
+ float_map_col MAP<FLOAT, FLOAT>,
+ double_map_col MAP<DOUBLE, DOUBLE>,
+ string_map_col MAP<STRING, STRING>,
+ boolean_map_col MAP<BOOLEAN, BOOLEAN>)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping"=":key,cf-tinyint:,cf-smallint:,cf-int:,cf-bigint:,cf-float:,cf-double:,cf-string:,cf-boolean:")
+TBLPROPERTIES (
+"hbase.table.name"="t_hive_maps_1",
+"hbase.table.default.storage.type"="binary");
+
+SELECT * FROM t_ext_hbase_maps_5 ORDER BY key;
+
+DROP TABLE t_ext_hbase_maps_5;
+
+DROP TABLE t_hbase_maps_1;
+
+DROP TABLE t_hbase_maps;
+
+DROP TABLE hbase_src;
Added: hive/trunk/hbase-handler/src/test/queries/hbase_binary_storage_queries.q
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/queries/hbase_binary_storage_queries.q?rev=1298673&view=auto
==============================================================================
--- hive/trunk/hbase-handler/src/test/queries/hbase_binary_storage_queries.q (added)
+++ hive/trunk/hbase-handler/src/test/queries/hbase_binary_storage_queries.q Fri Mar 9 00:47:37 2012
@@ -0,0 +1,218 @@
+DROP TABLE t_hbase;
+
+CREATE TABLE t_hbase(key STRING,
+ tinyint_col TINYINT,
+ smallint_col SMALLINT,
+ int_col INT,
+ bigint_col BIGINT,
+ float_col FLOAT,
+ double_col DOUBLE,
+ boolean_col BOOLEAN)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key#-,cf:binarybyte#-,cf:binaryshort#-,cf:binaryint#-,cf:binarylong#-,cf:binaryfloat#-,cf:binarydouble#-,cf:binaryboolean#-")
+TBLPROPERTIES ("hbase.table.name" = "t_hive",
+ "hbase.table.default.storage.type" = "binary");
+
+DESCRIBE FORMATTED t_hbase;
+
+INSERT OVERWRITE TABLE t_hbase
+SELECT 'user1', 1, 1, 1, 1, 1.0, 1.0, true
+FROM src
+WHERE key=100 OR key=125 OR key=126;
+
+INSERT OVERWRITE TABLE t_hbase
+SELECT 'user2', 127, 32767, 2147483647, 9223372036854775807, 211.31, 268746532.0571, false
+FROM src
+WHERE key=100 OR key=125 OR key=126;
+
+INSERT OVERWRITE TABLE t_hbase
+SELECT 'user3', -128, -32768, -2147483648, -9223372036854775808, -201.17, -2110789.37145, true
+FROM src
+WHERE key=100 OR key=125 OR key=126;
+
+SELECT * FROM t_hbase;
+
+SELECT tinyint_col,
+ smallint_col,
+ int_col,
+ bigint_col,
+ float_col,
+ double_col,
+ boolean_col
+FROM t_hbase
+WHERE key='user1' OR key='user2' OR key='user3';
+
+SELECT sum(tinyint_col),
+ sum(smallint_col),
+ sum(int_col),
+ sum(bigint_col),
+ sum(float_col),
+ sum(double_col),
+ count(boolean_col)
+FROM t_hbase;
+
+DROP TABLE t_hbase_1;
+
+CREATE EXTERNAL TABLE t_hbase_1(key STRING,
+ tinyint_col TINYINT,
+ smallint_col SMALLINT,
+ int_col INT,
+ bigint_col BIGINT,
+ float_col FLOAT,
+ double_col DOUBLE,
+ boolean_col BOOLEAN)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key#b,cf:binarybyte#b,cf:binaryshort#b,cf:binaryint#b,cf:binarylong#b,cf:binaryfloat#b,cf:binarydouble#b,cf:binaryboolean#b")
+TBLPROPERTIES ("hbase.table.name" = "t_hive");
+
+DESCRIBE FORMATTED t_hbase_1;
+
+SELECT * FROM t_hbase_1;
+
+SELECT tinyint_col,
+ smallint_col,
+ int_col,
+ bigint_col,
+ float_col,
+ double_col,
+ boolean_col
+FROM t_hbase_1
+WHERE key='user1' OR key='user2' OR key='user3';
+
+SELECT sum(tinyint_col),
+ sum(smallint_col),
+ sum(int_col),
+ sum(bigint_col),
+ sum(float_col),
+ sum(double_col),
+ count(boolean_col)
+FROM t_hbase_1;
+
+DROP TABLE t_hbase_1;
+DROP TABLE t_hbase;
+DROP TABLE t_hbase_2;
+
+CREATE TABLE t_hbase_2(key STRING,
+ tinyint_col TINYINT,
+ smallint_col SMALLINT,
+ int_col INT,
+ bigint_col BIGINT,
+ float_col FLOAT,
+ double_col DOUBLE,
+ boolean_col BOOLEAN)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key#-,cf:binarybyte#-,cf:binaryshort#-,cf:binaryint#-,cf:binarylong#-,cf:binaryfloat#-,cf:binarydouble#-,cf:binaryboolean#-")
+TBLPROPERTIES ("hbase.table.name" = "t_hive_2");
+
+INSERT OVERWRITE TABLE t_hbase_2
+SELECT 'user1', 1, 1, 1, 1, 1.0, 1.0, true
+FROM src
+WHERE key=100 OR key=125 OR key=126;
+
+INSERT OVERWRITE TABLE t_hbase_2
+SELECT 'user2', 127, 32767, 2147483647, 9223372036854775807, 211.31, 268746532.0571, false
+FROM src
+WHERE key=100 OR key=125 OR key=126;
+
+INSERT OVERWRITE TABLE t_hbase_2
+SELECT 'user3', -128, -32768, -2147483648, -9223372036854775808, -201.17, -2110789.37145, true
+FROM src
+WHERE key=100 OR key=125 OR key=126;
+
+SELECT * FROM t_hbase_2;
+
+SELECT tinyint_col,
+ smallint_col,
+ int_col,
+ bigint_col,
+ float_col,
+ double_col,
+ boolean_col
+FROM t_hbase_2
+WHERE key='user1' OR key='user2' OR key='user3';
+
+SELECT sum(tinyint_col),
+ sum(smallint_col),
+ sum(int_col),
+ sum(bigint_col),
+ sum(float_col),
+ sum(double_col),
+ count(boolean_col)
+FROM t_hbase_2;
+
+DROP TABLE t_hbase_3;
+
+CREATE EXTERNAL TABLE t_hbase_3(key STRING,
+ tinyint_col TINYINT,
+ smallint_col SMALLINT,
+ int_col INT,
+ bigint_col BIGINT,
+ float_col FLOAT,
+ double_col DOUBLE,
+ boolean_col BOOLEAN)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key#b,cf:binarybyte#b,cf:binaryshort#b,cf:binaryint#b,cf:binarylong#b,cf:binaryfloat#b,cf:binarydouble#b,cf:binaryboolean#b")
+TBLPROPERTIES ("hbase.table.name" = "t_hive_2");
+
+SELECT * FROM t_hbase_3;
+
+SELECT tinyint_col,
+ smallint_col,
+ int_col,
+ bigint_col,
+ float_col,
+ double_col,
+ boolean_col
+FROM t_hbase_3
+WHERE key='user1' OR key='user2' OR key='user3';
+
+SELECT sum(tinyint_col),
+ sum(smallint_col),
+ sum(int_col),
+ sum(bigint_col),
+ sum(float_col),
+ sum(double_col),
+ count(boolean_col)
+FROM t_hbase_3;
+
+DROP TABLE t_hbase_3;
+
+DROP TABLE t_hbase_4;
+
+CREATE EXTERNAL TABLE t_hbase_4(key STRING,
+ tinyint_col TINYINT,
+ smallint_col SMALLINT,
+ int_col INT,
+ bigint_col BIGINT,
+ float_col FLOAT,
+ double_col DOUBLE,
+ boolean_col BOOLEAN)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key#-,cf:binarybyte#-,cf:binaryshort#-,cf:binaryint#-,cf:binarylong#-,cf:binaryfloat#-,cf:binarydouble#-,cf:binaryboolean#-")
+TBLPROPERTIES (
+"hbase.table.name" = "t_hive_2",
+"hbase.table.default.storage.type" = "binary");
+
+SELECT * FROM t_hbase_4;
+
+SELECT tinyint_col,
+ smallint_col,
+ int_col,
+ bigint_col,
+ float_col,
+ double_col,
+ boolean_col
+FROM t_hbase_4
+WHERE key='user1' OR key='user2' OR key='user3';
+
+SELECT sum(tinyint_col),
+ sum(smallint_col),
+ sum(int_col),
+ sum(bigint_col),
+ sum(float_col),
+ sum(double_col),
+ count(boolean_col)
+FROM t_hbase_4;
+
+DROP TABLE t_hbase_4;
+DROP TABLE t_hbase_2;
Added: hive/trunk/hbase-handler/src/test/results/hbase_binary_external_table_queries.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/results/hbase_binary_external_table_queries.q.out?rev=1298673&view=auto
==============================================================================
--- hive/trunk/hbase-handler/src/test/results/hbase_binary_external_table_queries.q.out (added)
+++ hive/trunk/hbase-handler/src/test/results/hbase_binary_external_table_queries.q.out Fri Mar 9 00:47:37 2012
@@ -0,0 +1,118 @@
+PREHOOK: query: DROP TABLE t_ext_hbase_1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE t_ext_hbase_1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE EXTERNAL TABLE t_ext_hbase_1
+(key STRING, c_bool BOOLEAN, c_byte TINYINT, c_short SMALLINT,
+ c_int INT, c_long BIGINT, c_string STRING, c_float FLOAT, c_double DOUBLE)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:cq-boolean,cf:cq-byte,cf:cq-short,cf:cq-int,cf:cq-long,cf:cq-string,cf:cq-float,cf:cq-double")
+TBLPROPERTIES ("hbase.table.name" = "HiveExternalTable")
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE EXTERNAL TABLE t_ext_hbase_1
+(key STRING, c_bool BOOLEAN, c_byte TINYINT, c_short SMALLINT,
+ c_int INT, c_long BIGINT, c_string STRING, c_float FLOAT, c_double DOUBLE)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:cq-boolean,cf:cq-byte,cf:cq-short,cf:cq-int,cf:cq-long,cf:cq-string,cf:cq-float,cf:cq-double")
+TBLPROPERTIES ("hbase.table.name" = "HiveExternalTable")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@t_ext_hbase_1
+PREHOOK: query: SELECT * FROM t_ext_hbase_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t_ext_hbase_1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM t_ext_hbase_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t_ext_hbase_1
+#### A masked pattern was here ####
+key-1 NULL NULL NULL NULL NULL Hadoop, HBase, NULL NULL
+key-2 NULL NULL NULL NULL NULL Hive NULL NULL
+key-3 NULL NULL NULL NULL NULL Test Strings NULL NULL
+PREHOOK: query: DROP TABLE t_ext_hbase_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@t_ext_hbase_1
+PREHOOK: Output: default@t_ext_hbase_1
+POSTHOOK: query: DROP TABLE t_ext_hbase_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@t_ext_hbase_1
+POSTHOOK: Output: default@t_ext_hbase_1
+PREHOOK: query: DROP TABLE t_ext_hbase_2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE t_ext_hbase_2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE EXTERNAL TABLE t_ext_hbase_2
+(key STRING, c_bool BOOLEAN, c_byte TINYINT, c_short SMALLINT,
+ c_int INT, c_long BIGINT, c_string STRING, c_float FLOAT, c_double DOUBLE)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key#b,cf:cq-boolean#b,cf:cq-byte#b,cf:cq-short#b,cf:cq-int#b,cf:cq-long#b,cf:cq-string#b,cf:cq-float#b,cf:cq-double#b")
+TBLPROPERTIES ("hbase.table.name" = "HiveExternalTable")
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE EXTERNAL TABLE t_ext_hbase_2
+(key STRING, c_bool BOOLEAN, c_byte TINYINT, c_short SMALLINT,
+ c_int INT, c_long BIGINT, c_string STRING, c_float FLOAT, c_double DOUBLE)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key#b,cf:cq-boolean#b,cf:cq-byte#b,cf:cq-short#b,cf:cq-int#b,cf:cq-long#b,cf:cq-string#b,cf:cq-float#b,cf:cq-double#b")
+TBLPROPERTIES ("hbase.table.name" = "HiveExternalTable")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@t_ext_hbase_2
+PREHOOK: query: SELECT * FROM t_ext_hbase_2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t_ext_hbase_2
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM t_ext_hbase_2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t_ext_hbase_2
+#### A masked pattern was here ####
+key-1 true -128 -32768 -2147483648 -9223372036854775808 Hadoop, HBase, 1.4E-45 4.9E-324
+key-2 false -1 -1 -1 -1 Hive -1.0 -1.0
+key-3 true 127 32767 2147483647 9223372036854775807 Test Strings 3.4028235E38 1.7976931348623157E308
+PREHOOK: query: DROP TABLE t_ext_hbase_2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@t_ext_hbase_2
+PREHOOK: Output: default@t_ext_hbase_2
+POSTHOOK: query: DROP TABLE t_ext_hbase_2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@t_ext_hbase_2
+POSTHOOK: Output: default@t_ext_hbase_2
+PREHOOK: query: DROP TABLE t_ext_hbase_3
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE t_ext_hbase_3
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE EXTERNAL TABLE t_ext_hbase_3
+(key STRING, c_bool BOOLEAN, c_byte TINYINT, c_short SMALLINT,
+ c_int INT, c_long BIGINT, c_string STRING, c_float FLOAT, c_double DOUBLE)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:cq-boolean,cf:cq-byte,cf:cq-short,cf:cq-int,cf:cq-long,cf:cq-string,cf:cq-float,cf:cq-double")
+TBLPROPERTIES (
+"hbase.table.name" = "HiveExternalTable",
+"hbase.table.default.storage.type" = "binary")
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE EXTERNAL TABLE t_ext_hbase_3
+(key STRING, c_bool BOOLEAN, c_byte TINYINT, c_short SMALLINT,
+ c_int INT, c_long BIGINT, c_string STRING, c_float FLOAT, c_double DOUBLE)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:cq-boolean,cf:cq-byte,cf:cq-short,cf:cq-int,cf:cq-long,cf:cq-string,cf:cq-float,cf:cq-double")
+TBLPROPERTIES (
+"hbase.table.name" = "HiveExternalTable",
+"hbase.table.default.storage.type" = "binary")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@t_ext_hbase_3
+PREHOOK: query: SELECT * from t_ext_hbase_3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t_ext_hbase_3
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * from t_ext_hbase_3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t_ext_hbase_3
+#### A masked pattern was here ####
+key-1 true -128 -32768 -2147483648 -9223372036854775808 Hadoop, HBase, 1.4E-45 4.9E-324
+key-2 false -1 -1 -1 -1 Hive -1.0 -1.0
+key-3 true 127 32767 2147483647 9223372036854775807 Test Strings 3.4028235E38 1.7976931348623157E308
+PREHOOK: query: DROP table t_ext_hbase_3
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@t_ext_hbase_3
+PREHOOK: Output: default@t_ext_hbase_3
+POSTHOOK: query: DROP table t_ext_hbase_3
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@t_ext_hbase_3
+POSTHOOK: Output: default@t_ext_hbase_3