You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2015/09/21 20:00:03 UTC
hive git commit: HIVE-11849: NPE in HiveHBaseTableShapshotInputFormat
in query with just count(*) (Enis Soztutar via Jason Dere)
Repository: hive
Updated Branches:
refs/heads/master 4ff5b258c -> 2a65989a4
HIVE-11849: NPE in HiveHBaseTableShapshotInputFormat in query with just count(*) (Enis Soztutar via Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2a65989a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2a65989a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2a65989a
Branch: refs/heads/master
Commit: 2a65989a48e043404b7060296be8da9d3494911e
Parents: 4ff5b25
Author: Jason Dere <jd...@hortonworks.com>
Authored: Mon Sep 21 10:59:21 2015 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Mon Sep 21 10:59:21 2015 -0700
----------------------------------------------------------------------
.../HiveHBaseTableSnapshotInputFormat.java | 21 ++++++++++++-------
.../queries/positive/hbase_handler_snapshot.q | 4 ++++
.../positive/hbase_handler_snapshot.q.out | 22 ++++++++++++++++++++
3 files changed, 39 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/2a65989a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java
index 45e4de9..aa3a02f 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableSnapshotInputFormat.java
@@ -24,6 +24,9 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapred.TableInputFormat;
import org.apache.hadoop.hbase.mapred.TableSnapshotInputFormat;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
+import org.apache.hadoop.hbase.util.Base64;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.InputFormat;
@@ -41,15 +44,17 @@ public class HiveHBaseTableSnapshotInputFormat
TableSnapshotInputFormat delegate = new TableSnapshotInputFormat();
private static void setColumns(JobConf job) throws IOException {
- // hbase mapred API doesn't support scan at the moment.
Scan scan = HiveHBaseInputFormatUtil.getScan(job);
- byte[][] families = scan.getFamilies();
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < families.length; i++) {
- if (i > 0) sb.append(" ");
- sb.append(Bytes.toString(families[i]));
- }
- job.set(TableInputFormat.COLUMN_LIST, sb.toString());
+ job.set(org.apache.hadoop.hbase.mapreduce.TableInputFormat.SCAN,
+ convertScanToString(scan));
+ }
+
+ // TODO: Once HBASE-11163 is completed, use that API, or switch to
+ // using mapreduce version of the APIs. rather than mapred
+ // Copied from HBase's TableMapreduceUtil since it is not public API
+ static String convertScanToString(Scan scan) throws IOException {
+ ClientProtos.Scan proto = ProtobufUtil.toScan(scan);
+ return Base64.encodeBytes(proto.toByteArray());
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/2a65989a/hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q b/hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q
index 11d52fd..ebdc63c 100644
--- a/hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q
+++ b/hbase-handler/src/test/queries/positive/hbase_handler_snapshot.q
@@ -2,3 +2,7 @@ SET hive.hbase.snapshot.name=src_hbase_snapshot;
SET hive.hbase.snapshot.restoredir=/tmp;
SELECT * FROM src_hbase LIMIT 5;
+
+SELECT value FROM src_hbase LIMIT 5;
+
+select count(*) from src_hbase;
http://git-wip-us.apache.org/repos/asf/hive/blob/2a65989a/hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out b/hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out
index 1cb18b2..731646c 100644
--- a/hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_handler_snapshot.q.out
@@ -11,3 +11,25 @@ POSTHOOK: Input: default@src_hbase
100 val_100
103 val_103
104 val_104
+PREHOOK: query: SELECT value FROM src_hbase LIMIT 5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_hbase
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT value FROM src_hbase LIMIT 5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_hbase
+#### A masked pattern was here ####
+val_0
+val_10
+val_100
+val_103
+val_104
+PREHOOK: query: select count(*) from src_hbase
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_hbase
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from src_hbase
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_hbase
+#### A masked pattern was here ####
+309