You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by nz...@apache.org on 2010/06/04 01:08:08 UTC
svn commit: r951198 - in /hadoop/hive/trunk: ./
hbase-handler/src/java/org/apache/hadoop/hive/hbase/
hbase-handler/src/test/org/apache/hadoop/hive/hbase/
hbase-handler/src/test/queries/ hbase-handler/src/test/results/
Author: nzhang
Date: Thu Jun 3 23:08:08 2010
New Revision: 951198
URL: http://svn.apache.org/viewvc?rev=951198&view=rev
Log:
HIVE-1228. allow HBase key column to be anywhere in Hive table (John Sichi via Ning Zhang)
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java
hadoop/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
hadoop/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
hadoop/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java
hadoop/hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java
hadoop/hive/trunk/hbase-handler/src/test/queries/hbase_queries.q
hadoop/hive/trunk/hbase-handler/src/test/results/hbase_queries.q.out
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=951198&r1=951197&r2=951198&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Thu Jun 3 23:08:08 2010
@@ -203,6 +203,9 @@ Trunk - Unreleased
HIVE-1372. New algorithm for variance() UDAF
(Mayank Lahiri via jvs)
+ HIVE-1228. allow HBase key column to be anywhere in Hive table
+ (John Sichi via Ning Zhang)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java?rev=951198&r1=951197&r2=951198&view=diff
==============================================================================
--- hadoop/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java (original)
+++ hadoop/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java Thu Jun 3 23:08:08 2010
@@ -66,42 +66,28 @@ public class HBaseSerDe implements SerDe
public static final String HBASE_TABLE_NAME = "hbase.table.name";
+ public static final String HBASE_KEY_COL = ":key";
+
public static final Log LOG = LogFactory.getLog(
HBaseSerDe.class.getName());
private ObjectInspector cachedObjectInspector;
- private HBaseSerDeParameters hbSerDeParams;
+ private List<String> hbaseColumnNames;
+ private SerDeParameters serdeParams;
private boolean useJSONSerialize;
private LazyHBaseRow cachedHBaseRow;
- private BatchUpdate serializeCache;
private ByteStream.Output serializeStream = new ByteStream.Output();
+ private int iKey;
- /**
- * HBaseSerDeParameters defines the parameters used to
- * instantiate HBaseSerDe.
- */
- public static class HBaseSerDeParameters {
- private List<String> hbaseColumnNames;
- private SerDeParameters serdeParams;
-
- public List<String> getHBaseColumnNames() {
- return hbaseColumnNames;
- }
-
- public SerDeParameters getSerDeParameters() {
- return serdeParams;
- }
- }
-
public String toString() {
return getClass().toString()
+ "["
- + hbSerDeParams.hbaseColumnNames
+ + hbaseColumnNames
+ ":"
- + ((StructTypeInfo) hbSerDeParams.serdeParams.getRowTypeInfo())
+ + ((StructTypeInfo) serdeParams.getRowTypeInfo())
.getAllStructFieldNames()
+ ":"
- + ((StructTypeInfo) hbSerDeParams.serdeParams.getRowTypeInfo())
+ + ((StructTypeInfo) serdeParams.getRowTypeInfo())
.getAllStructFieldTypeInfos() + "]";
}
@@ -114,93 +100,104 @@ public class HBaseSerDe implements SerDe
*/
public void initialize(Configuration conf, Properties tbl)
throws SerDeException {
- hbSerDeParams = HBaseSerDe.initHBaseSerDeParameters(conf, tbl,
+
+ initHBaseSerDeParameters(conf, tbl,
getClass().getName());
- // We just used columnNames & columnTypes these two parameters
cachedObjectInspector = LazyFactory.createLazyStructInspector(
- hbSerDeParams.serdeParams.getColumnNames(),
- hbSerDeParams.serdeParams.getColumnTypes(),
- hbSerDeParams.serdeParams.getSeparators(),
- hbSerDeParams.serdeParams.getNullSequence(),
- hbSerDeParams.serdeParams.isLastColumnTakesRest(),
- hbSerDeParams.serdeParams.isEscaped(),
- hbSerDeParams.serdeParams.getEscapeChar());
+ serdeParams.getColumnNames(),
+ serdeParams.getColumnTypes(),
+ serdeParams.getSeparators(),
+ serdeParams.getNullSequence(),
+ serdeParams.isLastColumnTakesRest(),
+ serdeParams.isEscaped(),
+ serdeParams.getEscapeChar());
cachedHBaseRow = new LazyHBaseRow(
(LazySimpleStructObjectInspector) cachedObjectInspector);
if (LOG.isDebugEnabled()) {
LOG.debug("HBaseSerDe initialized with : columnNames = "
- + hbSerDeParams.serdeParams.getColumnNames()
+ + serdeParams.getColumnNames()
+ " columnTypes = "
- + hbSerDeParams.serdeParams.getColumnTypes()
+ + serdeParams.getColumnTypes()
+ " hbaseColumnMapping = "
- + hbSerDeParams.hbaseColumnNames);
+ + hbaseColumnNames);
}
}
+
+ public static List<String> parseColumnMapping(String columnMapping) {
+ String [] columnArray = columnMapping.split(",");
+ List<String> columnList = Arrays.asList(columnArray);
+ int iKey = columnList.indexOf(HBASE_KEY_COL);
+ if (iKey == -1) {
+ columnList = new ArrayList<String>(columnList);
+ columnList.add(0, HBASE_KEY_COL);
+ }
+ return columnList;
+ }
+
+ public static boolean isSpecialColumn(String hbaseColumnName) {
+ return hbaseColumnName.equals(HBASE_KEY_COL);
+ }
- public static HBaseSerDeParameters initHBaseSerDeParameters(
+ private void initHBaseSerDeParameters(
Configuration job, Properties tbl, String serdeName)
throws SerDeException {
- HBaseSerDeParameters serdeParams = new HBaseSerDeParameters();
-
- // Read Configuration Parameter
+ // Read configuration parameters
String hbaseColumnNameProperty =
tbl.getProperty(HBaseSerDe.HBASE_COL_MAPPING);
String columnTypeProperty =
tbl.getProperty(Constants.LIST_COLUMN_TYPES);
- // Initial the hbase column list
- if (hbaseColumnNameProperty != null
- && hbaseColumnNameProperty.length() > 0) {
-
- serdeParams.hbaseColumnNames =
- Arrays.asList(hbaseColumnNameProperty.split(","));
- } else {
- serdeParams.hbaseColumnNames = new ArrayList<String>();
- }
-
- // Add the hbase key to the columnNameList and columnTypeList
-
- // Build the type property string
+ // Initialize the hbase column list
+ hbaseColumnNames = parseColumnMapping(hbaseColumnNameProperty);
+ iKey = hbaseColumnNames.indexOf(HBASE_KEY_COL);
+
+ // Build the type property string if not supplied
if (columnTypeProperty == null) {
StringBuilder sb = new StringBuilder();
- sb.append(Constants.STRING_TYPE_NAME);
-
- for (int i = 0; i < serdeParams.hbaseColumnNames.size(); i++) {
- String colName = serdeParams.hbaseColumnNames.get(i);
- if (colName.endsWith(":")) {
- sb.append(":").append(
+
+ for (int i = 0; i < hbaseColumnNames.size(); i++) {
+ if (sb.length() > 0) {
+ sb.append(":");
+ }
+ String colName = hbaseColumnNames.get(i);
+ if (isSpecialColumn(colName)) {
+ // a special column becomes a STRING
+ sb.append(Constants.STRING_TYPE_NAME);
+ } else if (colName.endsWith(":")) {
+ // a column family become a MAP
+ sb.append(
Constants.MAP_TYPE_NAME + "<"
+ Constants.STRING_TYPE_NAME
+ "," + Constants.STRING_TYPE_NAME + ">");
} else {
- sb.append(":").append(Constants.STRING_TYPE_NAME);
+ // an individual column becomes a STRING
+ sb.append(Constants.STRING_TYPE_NAME);
}
}
tbl.setProperty(Constants.LIST_COLUMN_TYPES, sb.toString());
}
- serdeParams.serdeParams = LazySimpleSerDe.initSerdeParams(
+ serdeParams = LazySimpleSerDe.initSerdeParams(
job, tbl, serdeName);
- if (serdeParams.hbaseColumnNames.size() + 1
- != serdeParams.serdeParams.getColumnNames().size()) {
-
+ if (hbaseColumnNames.size() != serdeParams.getColumnNames().size()) {
throw new SerDeException(serdeName + ": columns has " +
- serdeParams.serdeParams.getColumnNames().size() +
- " elements while hbase.columns.mapping has " +
- serdeParams.hbaseColumnNames.size() + " elements!");
+ serdeParams.getColumnNames().size() +
+ " elements while hbase.columns.mapping has " +
+ hbaseColumnNames.size() + " elements" +
+ " (counting the key if implicit)");
}
// check that the mapping schema is right;
// we just can make sure that "columnfamily:" is mapped to MAP<String,?>
- for (int i = 0; i < serdeParams.hbaseColumnNames.size(); i++) {
- String hbaseColName = serdeParams.hbaseColumnNames.get(i);
- if (hbaseColName.endsWith(":")) {
- TypeInfo typeInfo = serdeParams.serdeParams.getColumnTypes().get(i + 1);
+ for (int i = 0; i < hbaseColumnNames.size(); i++) {
+ String hbaseColName = hbaseColumnNames.get(i);
+ if (hbaseColName.endsWith(":")) {
+ TypeInfo typeInfo = serdeParams.getColumnTypes().get(i);
if ((typeInfo.getCategory() != Category.MAP) ||
(((MapTypeInfo) typeInfo).getMapKeyTypeInfo().getTypeName()
!= Constants.STRING_TYPE_NAME)) {
@@ -213,8 +210,6 @@ public class HBaseSerDe implements SerDe
}
}
}
-
- return serdeParams;
}
/**
@@ -230,7 +225,7 @@ public class HBaseSerDe implements SerDe
}
RowResult rr = (RowResult)rowResult;
- cachedHBaseRow.init(rr, hbSerDeParams.hbaseColumnNames);
+ cachedHBaseRow.init(rr, hbaseColumnNames);
return cachedHBaseRow;
}
@@ -258,128 +253,145 @@ public class HBaseSerDe implements SerDe
List<? extends StructField> fields = soi.getAllStructFieldRefs();
List<Object> list = soi.getStructFieldsDataAsList(obj);
List<? extends StructField> declaredFields =
- (hbSerDeParams.serdeParams.getRowTypeInfo() != null &&
- ((StructTypeInfo) hbSerDeParams.serdeParams.getRowTypeInfo())
+ (serdeParams.getRowTypeInfo() != null &&
+ ((StructTypeInfo) serdeParams.getRowTypeInfo())
.getAllStructFieldNames().size() > 0) ?
((StructObjectInspector)getObjectInspector()).getAllStructFieldRefs()
: null;
- boolean isNotNull = false;
- String hbaseColumn = "";
+ BatchUpdate batchUpdate;
try {
+ byte [] key =
+ serializeField(
+ iKey, HBASE_KEY_COL, null, fields, list, declaredFields);
+ if (key == null) {
+ throw new SerDeException("HBase row key cannot be NULL");
+ }
+ batchUpdate = new BatchUpdate(key);
// Serialize each field
for (int i = 0; i < fields.size(); i++) {
- serializeStream.reset();
- // Get the field objectInspector and the field object.
- ObjectInspector foi = fields.get(i).getFieldObjectInspector();
- Object f = (list == null ? null : list.get(i));
-
- if (declaredFields != null && i >= declaredFields.size()) {
- throw new SerDeException(
- "Error: expecting " + declaredFields.size()
- + " but asking for field " + i + "\n" + "data=" + obj + "\n"
- + "tableType="
- + hbSerDeParams.serdeParams.getRowTypeInfo().toString()
- + "\n"
- + "dataType="
- + TypeInfoUtils.getTypeInfoFromObjectInspector(objInspector));
- }
-
- if (f == null) {
- // a null object, we do not serialize it
+ if (i == iKey) {
+ // already processed the key above
continue;
}
-
- if (i > 0) {
- hbaseColumn = hbSerDeParams.hbaseColumnNames.get(i-1);
- }
-
- // If the field that is column family in hbase
- if (i > 0 && hbaseColumn.endsWith(":")) {
- MapObjectInspector moi = (MapObjectInspector)foi;
- ObjectInspector koi = moi.getMapKeyObjectInspector();
- ObjectInspector voi = moi.getMapValueObjectInspector();
+ String hbaseColumn = hbaseColumnNames.get(i);
+ serializeField(
+ i, hbaseColumn, batchUpdate, fields, list, declaredFields);
+ }
+ } catch (IOException e) {
+ throw new SerDeException(e);
+ }
+
+ return batchUpdate;
+ }
- Map<?, ?> map = moi.getMap(f);
- if (map == null) {
- continue;
- } else {
- for (Map.Entry<?, ?> entry: map.entrySet()) {
- // Get the Key
- serialize(serializeStream, entry.getKey(), koi,
- hbSerDeParams.serdeParams.getSeparators(), 3,
- hbSerDeParams.serdeParams.getNullSequence(),
- hbSerDeParams.serdeParams.isEscaped(),
- hbSerDeParams.serdeParams.getEscapeChar(),
- hbSerDeParams.serdeParams.getNeedsEscape());
-
- // generate a column name (column_family:column_name)
- hbaseColumn += Bytes.toString(
- serializeStream.getData(), 0, serializeStream.getCount());
-
- // Get the Value
- serializeStream.reset();
-
- isNotNull = serialize(serializeStream, entry.getValue(), voi,
- hbSerDeParams.serdeParams.getSeparators(), 3,
- hbSerDeParams.serdeParams.getNullSequence(),
- hbSerDeParams.serdeParams.isEscaped(),
- hbSerDeParams.serdeParams.getEscapeChar(),
- hbSerDeParams.serdeParams.getNeedsEscape());
- }
- }
- } else {
- // If the field that is passed in is NOT a primitive, and either the
- // field is not declared (no schema was given at initialization), or
- // the field is declared as a primitive in initialization, serialize
- // the data to JSON string. Otherwise serialize the data in the
- // delimited way.
- if (!foi.getCategory().equals(Category.PRIMITIVE)
- && (declaredFields == null ||
- declaredFields.get(i).getFieldObjectInspector().getCategory()
- .equals(Category.PRIMITIVE) || useJSONSerialize)) {
- isNotNull = serialize(
- serializeStream, SerDeUtils.getJSONString(f, foi),
- PrimitiveObjectInspectorFactory.javaStringObjectInspector,
- hbSerDeParams.serdeParams.getSeparators(), 1,
- hbSerDeParams.serdeParams.getNullSequence(),
- hbSerDeParams.serdeParams.isEscaped(),
- hbSerDeParams.serdeParams.getEscapeChar(),
- hbSerDeParams.serdeParams.getNeedsEscape());
- } else {
- isNotNull = serialize(
- serializeStream, f, foi,
- hbSerDeParams.serdeParams.getSeparators(), 1,
- hbSerDeParams.serdeParams.getNullSequence(),
- hbSerDeParams.serdeParams.isEscaped(),
- hbSerDeParams.serdeParams.getEscapeChar(),
- hbSerDeParams.serdeParams.getNeedsEscape());
- }
- }
+ private byte [] serializeField(
+ int i, String hbaseColumn, BatchUpdate batchUpdate,
+ List<? extends StructField> fields,
+ List<Object> list,
+ List<? extends StructField> declaredFields) throws IOException {
+
+ // Get the field objectInspector and the field object.
+ ObjectInspector foi = fields.get(i).getFieldObjectInspector();
+ Object f = (list == null ? null : list.get(i));
+
+ if (f == null) {
+ // a null object, we do not serialize it
+ return null;
+ }
- byte [] key = new byte[serializeStream.getCount()];
- System.arraycopy(
- serializeStream.getData(), 0, key, 0, serializeStream.getCount());
- if (i == 0) {
- // the first column is the hbase key
- serializeCache = new BatchUpdate(key);
- } else {
- if (isNotNull) {
- serializeCache.put(hbaseColumn, key);
+ // If the field corresponds to a column family in hbase
+ if (hbaseColumn.endsWith(":")) {
+ MapObjectInspector moi = (MapObjectInspector)foi;
+ ObjectInspector koi = moi.getMapKeyObjectInspector();
+ ObjectInspector voi = moi.getMapValueObjectInspector();
+
+ Map<?, ?> map = moi.getMap(f);
+ if (map == null) {
+ return null;
+ } else {
+ for (Map.Entry<?, ?> entry: map.entrySet()) {
+ // Get the Key
+ serializeStream.reset();
+ serialize(serializeStream, entry.getKey(), koi,
+ serdeParams.getSeparators(), 3,
+ serdeParams.getNullSequence(),
+ serdeParams.isEscaped(),
+ serdeParams.getEscapeChar(),
+ serdeParams.getNeedsEscape());
+
+ // generate a column name (column_family:column_name)
+ String hbaseSparseColumn =
+ hbaseColumn + Bytes.toString(
+ serializeStream.getData(), 0, serializeStream.getCount());
+
+ // Get the Value
+ serializeStream.reset();
+
+ boolean isNotNull = serialize(serializeStream, entry.getValue(), voi,
+ serdeParams.getSeparators(), 3,
+ serdeParams.getNullSequence(),
+ serdeParams.isEscaped(),
+ serdeParams.getEscapeChar(),
+ serdeParams.getNeedsEscape());
+ if (!isNotNull) {
+ continue;
}
+ byte [] key = new byte[serializeStream.getCount()];
+ System.arraycopy(
+ serializeStream.getData(), 0, key, 0, serializeStream.getCount());
+ batchUpdate.put(hbaseSparseColumn, key);
}
}
- } catch (IOException e) {
- throw new SerDeException(e);
+ } else {
+ // If the field that is passed in is NOT a primitive, and either the
+ // field is not declared (no schema was given at initialization), or
+ // the field is declared as a primitive in initialization, serialize
+ // the data to JSON string. Otherwise serialize the data in the
+ // delimited way.
+ serializeStream.reset();
+ boolean isNotNull;
+ if (!foi.getCategory().equals(Category.PRIMITIVE)
+ && (declaredFields == null ||
+ declaredFields.get(i).getFieldObjectInspector().getCategory()
+ .equals(Category.PRIMITIVE) || useJSONSerialize)) {
+ isNotNull = serialize(
+ serializeStream, SerDeUtils.getJSONString(f, foi),
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector,
+ serdeParams.getSeparators(), 1,
+ serdeParams.getNullSequence(),
+ serdeParams.isEscaped(),
+ serdeParams.getEscapeChar(),
+ serdeParams.getNeedsEscape());
+ } else {
+ isNotNull = serialize(
+ serializeStream, f, foi,
+ serdeParams.getSeparators(), 1,
+ serdeParams.getNullSequence(),
+ serdeParams.isEscaped(),
+ serdeParams.getEscapeChar(),
+ serdeParams.getNeedsEscape());
+ }
+ if (!isNotNull) {
+ return null;
+ }
+ byte [] key = new byte[serializeStream.getCount()];
+ System.arraycopy(
+ serializeStream.getData(), 0, key, 0, serializeStream.getCount());
+ if (hbaseColumn.equals(HBASE_KEY_COL)) {
+ return key;
+ }
+ batchUpdate.put(hbaseColumn, key);
}
-
- return serializeCache;
+
+ return null;
}
/**
- * Serialize the row into the StringBuilder.
- * @param out The StringBuilder to store the serialized data.
+ * Serialize the row into a ByteStream.
+ *
+ * @param out The ByteStream.Output to store the serialized data.
* @param obj The object for the current field.
* @param objInspector The ObjectInspector for the current Object.
* @param separators The separators array.
@@ -460,7 +472,7 @@ public class HBaseSerDe implements SerDe
if (list == null) {
return false;
} else {
- for (int i = 0; i<list.size(); i++) {
+ for (int i = 0; i < list.size(); i++) {
if (i > 0) {
out.write(separator);
}
Modified: hadoop/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java?rev=951198&r1=951197&r2=951198&view=diff
==============================================================================
--- hadoop/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java (original)
+++ hadoop/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseStorageHandler.java Thu Jun 3 23:08:08 2010
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.hbase;
import java.io.IOException;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
@@ -133,8 +134,17 @@ public class HBaseStorageHandler
if (hbaseColumnStr == null) {
throw new MetaException("No hbase.columns.mapping defined in Serde.");
}
- String [] hbaseColumns = hbaseColumnStr.split(",");
+ List<String> hbaseColumns =
+ HBaseSerDe.parseColumnMapping(hbaseColumnStr);
+ int iKeyFirst = hbaseColumns.indexOf(HBaseSerDe.HBASE_KEY_COL);
+ int iKeyLast = hbaseColumns.lastIndexOf(HBaseSerDe.HBASE_KEY_COL);
+ if (iKeyFirst != iKeyLast) {
+ throw new MetaException("Multiple key columns defined in hbase.columns.mapping.");
+ }
for (String hbaseColumn : hbaseColumns) {
+ if (HBaseSerDe.isSpecialColumn(hbaseColumn)) {
+ continue;
+ }
int idx = hbaseColumn.indexOf(":");
if (idx < 0) {
throw new MetaException(
Modified: hadoop/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java?rev=951198&r1=951197&r2=951198&view=diff
==============================================================================
--- hadoop/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java (original)
+++ hadoop/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java Thu Jun 3 23:08:08 2010
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.hbase;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -70,51 +71,40 @@ public class HiveHBaseTableInputFormat<K
new HBaseConfiguration(job),
Bytes.toBytes(hbaseTableName)));
- // because the hbase key is mapped to the first column in its hive table,
- // we add the "_key" before the columnMapping that we can use the
- // hive column id to find the exact hbase column one-for-one.
- String columnMapping = "_key," + hbaseSplit.getColumnsMapping();
- String[] columns = columnMapping.split(",");
+ String columnMapping = hbaseSplit.getColumnsMapping();
+ List<String> columns = HBaseSerDe.parseColumnMapping(columnMapping);
List<Integer> readColIDs =
ColumnProjectionUtils.getReadColumnIDs(job);
- if (columns.length < readColIDs.size()) {
+ if (columns.size() < readColIDs.size()) {
throw new IOException(
"Cannot read more columns than the given table contains.");
}
-
- byte [][] scanColumns;
- if (readColIDs.size() == 0) {
- scanColumns = new byte[columns.length - 1][];
- for (int i=0; i < columns.length - 1; i++) {
- scanColumns[i] = Bytes.toBytes(columns[i + 1]);
+
+ List<byte []> scanColumns = new ArrayList<byte []>();
+ boolean addAll = (readColIDs.size() == 0);
+ if (!addAll) {
+ for (int iColumn : readColIDs) {
+ String column = columns.get(iColumn);
+ if (HBaseSerDe.isSpecialColumn(column)) {
+ continue;
+ }
+ scanColumns.add(Bytes.toBytes(column));
}
- } else {
- Collections.sort(readColIDs);
-
- if (readColIDs.get(0) == 0) {
- // sql like "select key from hbasetable;"
- // As HBase can not scan a hbase table while just getting its keys,
- // so we will scan out the second column of the hive table
- // but ignore it.
- if (readColIDs.size() == 1) {
- scanColumns = new byte[1][];
- scanColumns[0] = Bytes.toBytes(columns[1]);
- } else {
- scanColumns = new byte[readColIDs.size() - 1][];
- for (int i=0; i<scanColumns.length; i++) {
- scanColumns[i] = Bytes.toBytes(columns[readColIDs.get(i + 1)]);
- }
+ }
+ if (scanColumns.isEmpty()) {
+ for (String column : columns) {
+ if (HBaseSerDe.isSpecialColumn(column)) {
+ continue;
}
- } else {
- scanColumns = new byte[readColIDs.size()][];
- for (int i=0; i<scanColumns.length; i++) {
- scanColumns[i] = Bytes.toBytes(columns[readColIDs.get(i)]);
+ scanColumns.add(Bytes.toBytes(column));
+ if (!addAll) {
+ break;
}
}
}
- hbaseInputFormat.setScanColumns(scanColumns);
+ hbaseInputFormat.setScanColumns(scanColumns.toArray(new byte[0][]));
return (RecordReader<K, V>)
hbaseInputFormat.getRecordReader(hbaseSplit.getSplit(), job, reporter);
@@ -131,14 +121,19 @@ public class HiveHBaseTableInputFormat<K
if (hbaseSchemaMapping == null) {
throw new IOException("hbase.columns.mapping required for HBase Table.");
}
-
- String [] columns = hbaseSchemaMapping.split(",");
- byte [][] inputColumns = new byte[columns.length][];
- for (int i=0; i < columns.length; i++) {
- inputColumns[i] = Bytes.toBytes(columns[i]);
+
+ // REVIEW: are we supposed to be applying the getReadColumnIDs
+ // same as in getRecordReader?
+ List<String> columns = HBaseSerDe.parseColumnMapping(hbaseSchemaMapping);
+ List<byte []> inputColumns = new ArrayList<byte []>();
+ for (String column : columns) {
+ if (HBaseSerDe.isSpecialColumn(column)) {
+ continue;
+ }
+ inputColumns.add(Bytes.toBytes(column));
}
- hbaseInputFormat.setScanColumns(inputColumns);
+ hbaseInputFormat.setScanColumns(inputColumns.toArray(new byte[0][]));
InputSplit[] splits = hbaseInputFormat.getSplits(
job, numSplits <= 0 ? 1 : numSplits);
Modified: hadoop/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java?rev=951198&r1=951197&r2=951198&view=diff
==============================================================================
--- hadoop/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java (original)
+++ hadoop/hive/trunk/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java Thu Jun 3 23:08:08 2010
@@ -71,16 +71,14 @@ public class LazyHBaseRow extends LazySt
((StructObjectInspector)getInspector()).getAllStructFieldRefs();
setFields(new LazyObject[fieldRefs.size()]);
for (int i = 0; i < getFields().length; i++) {
- if (i > 0) {
- String hbaseColumn = hbaseColumns.get(i - 1);
- if (hbaseColumn.endsWith(":")) {
- // a column family
- getFields()[i] =
- new LazyHBaseCellMap(
- (LazyMapObjectInspector)
- fieldRefs.get(i).getFieldObjectInspector());
- continue;
- }
+ String hbaseColumn = hbaseColumns.get(i);
+ if (hbaseColumn.endsWith(":")) {
+ // a column family
+ getFields()[i] =
+ new LazyHBaseCellMap(
+ (LazyMapObjectInspector)
+ fieldRefs.get(i).getFieldObjectInspector());
+ continue;
}
getFields()[i] = LazyFactory.createLazyObject(
@@ -122,14 +120,13 @@ public class LazyHBaseRow extends LazySt
if (!getFieldInited()[fieldID]) {
getFieldInited()[fieldID] = true;
- ByteArrayRef ref = new ByteArrayRef();
+ ByteArrayRef ref = null;
- if (fieldID == 0) {
- // the key
+ String columnName = hbaseColumns.get(fieldID);
+ if (columnName.equals(HBaseSerDe.HBASE_KEY_COL)) {
+ ref = new ByteArrayRef();
ref.setData(rowResult.getRow());
- getFields()[fieldID].init(ref, 0, ref.getData().length);
} else {
- String columnName = hbaseColumns.get(fieldID - 1);
if (columnName.endsWith(":")) {
// it is a column family
((LazyHBaseCellMap) getFields()[fieldID]).init(
@@ -137,13 +134,16 @@ public class LazyHBaseRow extends LazySt
} else {
// it is a column
if (rowResult.containsKey(columnName)) {
+ ref = new ByteArrayRef();
ref.setData(rowResult.get(columnName).getValue());
- getFields()[fieldID].init(ref, 0, ref.getData().length);
} else {
return null;
}
}
}
+ if (ref != null) {
+ getFields()[fieldID].init(ref, 0, ref.getData().length);
+ }
}
return getFields()[fieldID].getObject();
}
Modified: hadoop/hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java?rev=951198&r1=951197&r2=951198&view=diff
==============================================================================
--- hadoop/hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java (original)
+++ hadoop/hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseObject.java Thu Jun 3 23:08:08 2010
@@ -145,7 +145,7 @@ public class TestLazyHBaseObject extends
Text nullSequence = new Text("\\N");
List<String> hbaseColumnNames =
- Arrays.asList(new String[]{"cfa:a", "cfa:b", "cfb:c", "cfb:d"});
+ Arrays.asList(new String[]{":key", "cfa:a", "cfa:b", "cfb:c", "cfb:d"});
ObjectInspector oi = LazyFactory.createLazyStructInspector(fieldNames,
fieldTypeInfos, new byte[] {' ', ':', '='},
@@ -223,7 +223,7 @@ public class TestLazyHBaseObject extends
Text nullSequence = new Text("\\N");
List<String> hbaseColumnNames =
- Arrays.asList(new String[]{"cfa:a", "cfa:b", "cfb:", "cfc:d"});
+ Arrays.asList(new String[]{":key", "cfa:a", "cfa:b", "cfb:", "cfc:d"});
ObjectInspector oi = LazyFactory.createLazyStructInspector(
fieldNames,
Modified: hadoop/hive/trunk/hbase-handler/src/test/queries/hbase_queries.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/hbase-handler/src/test/queries/hbase_queries.q?rev=951198&r1=951197&r2=951198&view=diff
==============================================================================
--- hadoop/hive/trunk/hbase-handler/src/test/queries/hbase_queries.q (original)
+++ hadoop/hive/trunk/hbase-handler/src/test/queries/hbase_queries.q Thu Jun 3 23:08:08 2010
@@ -115,18 +115,31 @@ DROP TABLE hbase_table_6;
CREATE TABLE hbase_table_6(key int, value map<string,string>)
STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
WITH SERDEPROPERTIES (
-"hbase.columns.mapping" = "cf:"
+"hbase.columns.mapping" = ":key,cf:"
);
INSERT OVERWRITE TABLE hbase_table_6 SELECT key, map(value, key) FROM src
WHERE key=98 OR key=100;
SELECT * FROM hbase_table_6 ORDER BY key;
+DROP TABLE hbase_table_7;
+CREATE TABLE hbase_table_7(value map<string,string>, key int)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "cf:,:key"
+);
+INSERT OVERWRITE TABLE hbase_table_7
+SELECT map(value, key, upper(value), key+1), key FROM src
+WHERE key=98 OR key=100;
+
+SELECT * FROM hbase_table_7 ORDER BY key;
+
DROP TABLE hbase_table_1;
DROP TABLE hbase_table_2;
DROP TABLE hbase_table_3;
DROP TABLE hbase_table_4;
DROP TABLE hbase_table_5;
DROP TABLE hbase_table_6;
+DROP TABLE hbase_table_7;
DROP TABLE empty_hbase_table;
DROP TABLE empty_normal_table;
Modified: hadoop/hive/trunk/hbase-handler/src/test/results/hbase_queries.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/hbase-handler/src/test/results/hbase_queries.q.out?rev=951198&r1=951197&r2=951198&view=diff
==============================================================================
--- hadoop/hive/trunk/hbase-handler/src/test/results/hbase_queries.q.out (original)
+++ hadoop/hive/trunk/hbase-handler/src/test/results/hbase_queries.q.out Thu Jun 3 23:08:08 2010
@@ -20,15 +20,15 @@ POSTHOOK: type: DESCTABLE
key int from deserializer
value string from deserializer
-Detailed Table Information Table(tableName:hbase_table_1, dbName:default, owner:jsichi, createTime:1269035280, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:int, comment:null), FieldSchema(name:value, type:string, comment:null)], location:file:/Users/jsichi/open/hive-trunk/build/hbase-handler/test/data/warehouse/hbase_table_1, inputFormat:org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat, outputFormat:org.apache.hadoop.hive.hbase.HiveHBaseTableOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.hbase.HBaseSerDe, parameters:{serialization.format=1,hbase.columns.mapping=cf:string}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{hbase.table.name=hbase_table_0,transient_lastDdlTime=1269035280,storage_handler=org.apache.hadoop.hive.hbase.HBaseStorageHandler}, viewOriginalText:null, viewExpandedText:null, tableType:MANAGED_TABLE)
+Detailed Table Information Table(tableName:hbase_table_1, dbName:default, owner:jsichi, createTime:1273696968, lastAccessTime:0, retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:int, comment:null), FieldSchema(name:value, type:string, comment:null)], location:file:/Users/jsichi/open/hive-trunk/build/hbase-handler/test/data/warehouse/hbase_table_1, inputFormat:org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat, outputFormat:org.apache.hadoop.hive.hbase.HiveHBaseTableOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.hbase.HBaseSerDe, parameters:{serialization.format=1,hbase.columns.mapping=cf:string}), bucketCols:[], sortCols:[], parameters:{}), partitionKeys:[], parameters:{hbase.table.name=hbase_table_0,transient_lastDdlTime=1273696968,storage_handler=org.apache.hadoop.hive.hbase.HBaseStorageHandler}, viewOriginalText:null, viewExpandedText:null, tableType:MANAGED_TABLE)
PREHOOK: query: select * from hbase_table_1
PREHOOK: type: QUERY
PREHOOK: Input: default@hbase_table_1
-PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-48-00_358_2930240620841842050/10000
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-42-48_571_1567002521290917375/10000
POSTHOOK: query: select * from hbase_table_1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@hbase_table_1
-POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-48-00_358_2930240620841842050/10000
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-42-48_571_1567002521290917375/10000
PREHOOK: query: EXPLAIN FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT * WHERE (key%2)=0
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT * WHERE (key%2)=0
@@ -198,7 +198,7 @@ STAGE PLANS:
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
- file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-48-06_629_7030239162593413579/10002
+ file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-42-53_771_3408249171250779075/10002
Reduce Output Operator
key expressions:
expr: _col0
@@ -237,7 +237,7 @@ ORDER BY key, value LIMIT 20
PREHOOK: type: QUERY
PREHOOK: Input: default@hbase_table_1
PREHOOK: Input: default@src
-PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-48-07_673_8894235303094199068/10000
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-42-54_376_2770959312857635153/10000
POSTHOOK: query: SELECT Y.*
FROM
(SELECT hbase_table_1.* FROM hbase_table_1) x
@@ -248,7 +248,7 @@ ORDER BY key, value LIMIT 20
POSTHOOK: type: QUERY
POSTHOOK: Input: default@hbase_table_1
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-48-07_673_8894235303094199068/10000
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-42-54_376_2770959312857635153/10000
0 val_0
0 val_0
0 val_0
@@ -382,7 +382,7 @@ STAGE PLANS:
Stage: Stage-2
Map Reduce
Alias -> Map Operator Tree:
- file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-48-16_880_9199961230368056420/10002
+ file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-43-03_629_6066743056712305489/10002
Reduce Output Operator
key expressions:
expr: _col0
@@ -420,7 +420,7 @@ ORDER BY key,value
PREHOOK: type: QUERY
PREHOOK: Input: default@hbase_table_2
PREHOOK: Input: default@hbase_table_1
-PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-48-17_672_6345535566739909217/10000
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-43-04_169_137119299474143401/10000
POSTHOOK: query: SELECT Y.*
FROM
(SELECT hbase_table_1.* FROM hbase_table_1 WHERE hbase_table_1.key > 100) x
@@ -431,7 +431,7 @@ ORDER BY key,value
POSTHOOK: type: QUERY
POSTHOOK: Input: default@hbase_table_2
POSTHOOK: Input: default@hbase_table_1
-POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-48-17_672_6345535566739909217/10000
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-43-04_169_137119299474143401/10000
104 val_104
114 val_114
116 val_116
@@ -462,48 +462,48 @@ PREHOOK: query: select * from (select co
PREHOOK: type: QUERY
PREHOOK: Input: default@empty_hbase_table
PREHOOK: Input: default@empty_normal_table
-PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-48-30_055_1946406021397828675/10000
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-43-16_942_5359613916717018216/10000
POSTHOOK: query: select * from (select count(1) as c from empty_normal_table union all select count(1) as c from empty_hbase_table) x order by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@empty_hbase_table
POSTHOOK: Input: default@empty_normal_table
-POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-48-30_055_1946406021397828675/10000
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-43-16_942_5359613916717018216/10000
0
0
PREHOOK: query: select * from (select count(1) c from empty_normal_table union all select count(1) as c from hbase_table_1) x order by c
PREHOOK: type: QUERY
PREHOOK: Input: default@empty_normal_table
PREHOOK: Input: default@hbase_table_1
-PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-48-50_042_5680510697138513644/10000
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-43-30_013_1128777795950375727/10000
POSTHOOK: query: select * from (select count(1) c from empty_normal_table union all select count(1) as c from hbase_table_1) x order by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@empty_normal_table
POSTHOOK: Input: default@hbase_table_1
-POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-48-50_042_5680510697138513644/10000
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-43-30_013_1128777795950375727/10000
0
155
PREHOOK: query: select * from (select count(1) c from src union all select count(1) as c from empty_hbase_table) x order by c
PREHOOK: type: QUERY
PREHOOK: Input: default@empty_hbase_table
PREHOOK: Input: default@src
-PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-49-09_786_4512530963467426096/10000
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-43-42_378_1951769354517025095/10000
POSTHOOK: query: select * from (select count(1) c from src union all select count(1) as c from empty_hbase_table) x order by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@empty_hbase_table
POSTHOOK: Input: default@src
-POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-49-09_786_4512530963467426096/10000
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-43-42_378_1951769354517025095/10000
0
500
PREHOOK: query: select * from (select count(1) c from src union all select count(1) as c from hbase_table_1) x order by c
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Input: default@hbase_table_1
-PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-49-25_808_1089193588207337323/10000
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-43-55_822_8910566457500025591/10000
POSTHOOK: query: select * from (select count(1) c from src union all select count(1) as c from hbase_table_1) x order by c
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Input: default@hbase_table_1
-POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-49-25_808_1089193588207337323/10000
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-43-55_822_8910566457500025591/10000
155
500
PREHOOK: query: CREATE TABLE hbase_table_3(key int, value string, count int)
@@ -703,20 +703,20 @@ POSTHOOK: Output: default@hbase_table_3
PREHOOK: query: select count(1) from hbase_table_3
PREHOOK: type: QUERY
PREHOOK: Input: default@hbase_table_3
-PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-50-01_188_3927696898547656442/10000
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-44-21_943_5380732604060686051/10000
POSTHOOK: query: select count(1) from hbase_table_3
POSTHOOK: type: QUERY
POSTHOOK: Input: default@hbase_table_3
-POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-50-01_188_3927696898547656442/10000
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-44-21_943_5380732604060686051/10000
155
PREHOOK: query: select * from hbase_table_3 order by key, value limit 5
PREHOOK: type: QUERY
PREHOOK: Input: default@hbase_table_3
-PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-50-05_328_16296194898070452/10000
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-44-27_368_350478834712988449/10000
POSTHOOK: query: select * from hbase_table_3 order by key, value limit 5
POSTHOOK: type: QUERY
POSTHOOK: Input: default@hbase_table_3
-POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-50-05_328_16296194898070452/10000
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-44-27_368_350478834712988449/10000
0 val_0 3
2 val_2 1
4 val_4 1
@@ -725,11 +725,11 @@ POSTHOOK: Output: file:/Users/jsichi/ope
PREHOOK: query: select key, count from hbase_table_3 order by key, count desc limit 5
PREHOOK: type: QUERY
PREHOOK: Input: default@hbase_table_3
-PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-50-11_322_7398887165683623890/10000
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-44-32_965_1538708553566035002/10000
POSTHOOK: query: select key, count from hbase_table_3 order by key, count desc limit 5
POSTHOOK: type: QUERY
POSTHOOK: Input: default@hbase_table_3
-POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-50-11_322_7398887165683623890/10000
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-44-32_965_1538708553566035002/10000
0 3
2 1
4 1
@@ -765,11 +765,11 @@ POSTHOOK: Output: default@hbase_table_4
PREHOOK: query: SELECT * FROM hbase_table_4 ORDER BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@hbase_table_4
-PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-50-23_988_5167848005094793844/10000
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-44-44_558_7005505662989774902/10000
POSTHOOK: query: SELECT * FROM hbase_table_4 ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@hbase_table_4
-POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-50-23_988_5167848005094793844/10000
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-44-44_558_7005505662989774902/10000
98 val_98 99 100
100 val_100 101 102
PREHOOK: query: DROP TABLE hbase_table_5
@@ -790,11 +790,11 @@ POSTHOOK: Output: default@hbase_table_5
PREHOOK: query: SELECT * FROM hbase_table_5 ORDER BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@hbase_table_5
-PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-50-31_199_5870817343406163372/10000
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-44-49_993_55354442785403380/10000
POSTHOOK: query: SELECT * FROM hbase_table_5 ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@hbase_table_5
-POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-50-31_199_5870817343406163372/10000
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-44-49_993_55354442785403380/10000
98 {"b":"val_98","c":"99"}
100 {"b":"val_100","c":"101"}
PREHOOK: query: DROP TABLE hbase_table_6
@@ -804,13 +804,13 @@ POSTHOOK: type: DROPTABLE
PREHOOK: query: CREATE TABLE hbase_table_6(key int, value map<string,string>)
STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
WITH SERDEPROPERTIES (
-"hbase.columns.mapping" = "cf:"
+"hbase.columns.mapping" = ":key,cf:"
)
PREHOOK: type: CREATETABLE
POSTHOOK: query: CREATE TABLE hbase_table_6(key int, value map<string,string>)
STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
WITH SERDEPROPERTIES (
-"hbase.columns.mapping" = "cf:"
+"hbase.columns.mapping" = ":key,cf:"
)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: default@hbase_table_6
@@ -827,13 +827,52 @@ POSTHOOK: Output: default@hbase_table_6
PREHOOK: query: SELECT * FROM hbase_table_6 ORDER BY key
PREHOOK: type: QUERY
PREHOOK: Input: default@hbase_table_6
-PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-50-45_033_2381570055437756000/10000
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-45-01_880_6891496336862307220/10000
POSTHOOK: query: SELECT * FROM hbase_table_6 ORDER BY key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@hbase_table_6
-POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-03-19_14-50-45_033_2381570055437756000/10000
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-45-01_880_6891496336862307220/10000
98 {"val_98":"98"}
100 {"val_100":"100"}
+PREHOOK: query: DROP TABLE hbase_table_7
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE hbase_table_7
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE hbase_table_7(value map<string,string>, key int)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "cf:,:key"
+)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE hbase_table_7(value map<string,string>, key int)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES (
+"hbase.columns.mapping" = "cf:,:key"
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@hbase_table_7
+PREHOOK: query: INSERT OVERWRITE TABLE hbase_table_7
+SELECT map(value, key, upper(value), key+1), key FROM src
+WHERE key=98 OR key=100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_7
+POSTHOOK: query: INSERT OVERWRITE TABLE hbase_table_7
+SELECT map(value, key, upper(value), key+1), key FROM src
+WHERE key=98 OR key=100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_7
+PREHOOK: query: SELECT * FROM hbase_table_7 ORDER BY key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_table_7
+PREHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-45-13_100_5693137614419238592/10000
+POSTHOOK: query: SELECT * FROM hbase_table_7 ORDER BY key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_table_7
+POSTHOOK: Output: file:/Users/jsichi/open/hive-trunk/build/hbase-handler/scratchdir/hive_2010-05-12_13-45-13_100_5693137614419238592/10000
+{"VAL_98":"99.0","val_98":"98"} 98
+{"VAL_100":"101.0","val_100":"100"} 100
PREHOOK: query: DROP TABLE hbase_table_1
PREHOOK: type: DROPTABLE
POSTHOOK: query: DROP TABLE hbase_table_1
@@ -864,6 +903,11 @@ PREHOOK: type: DROPTABLE
POSTHOOK: query: DROP TABLE hbase_table_6
POSTHOOK: type: DROPTABLE
POSTHOOK: Output: default@hbase_table_6
+PREHOOK: query: DROP TABLE hbase_table_7
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE hbase_table_7
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@hbase_table_7
PREHOOK: query: DROP TABLE empty_hbase_table
PREHOOK: type: DROPTABLE
POSTHOOK: query: DROP TABLE empty_hbase_table