You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ai...@apache.org on 2018/02/12 19:03:34 UTC
hive git commit: HIVE-18550: Keep the hbase table name property as
hbase.table.name (Aihua Xu, reviewed by Yongzhi Chen)
Repository: hive
Updated Branches:
refs/heads/master fa14a4365 -> 1eddbc06a
HIVE-18550: Keep the hbase table name property as hbase.table.name (Aihua Xu, reviewed by Yongzhi Chen)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1eddbc06
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1eddbc06
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1eddbc06
Branch: refs/heads/master
Commit: 1eddbc06a6224cb860ecb2f331cb2462a57b228b
Parents: fa14a43
Author: Aihua Xu <ai...@apache.org>
Authored: Fri Jan 26 15:30:52 2018 -0800
Committer: Aihua Xu <ai...@apache.org>
Committed: Mon Feb 12 11:03:25 2018 -0800
----------------------------------------------------------------------
.../hadoop/hive/hbase/HiveHFileOutputFormat.java | 14 +++++++++++++-
hbase-handler/src/test/queries/positive/hbase_bulk.q | 2 +-
.../src/test/queries/positive/hbase_handler_bulk.q | 4 ++--
.../test/results/positive/hbase_handler_bulk.q.out | 8 ++++----
4 files changed, 20 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1eddbc06/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHFileOutputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHFileOutputFormat.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHFileOutputFormat.java
index d8dad06..4fa0272 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHFileOutputFormat.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHFileOutputFormat.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
import org.apache.hadoop.hive.shims.ShimLoader;
@@ -64,7 +65,8 @@ public class HiveHFileOutputFormat extends
HiveOutputFormat<ImmutableBytesWritable, Cell> {
public static final String HFILE_FAMILY_PATH = "hfile.family.path";
-
+ public static final String OUTPUT_TABLE_NAME_CONF_KEY =
+ "hbase.mapreduce.hfileoutputformat.table.name";
static final Logger LOG = LoggerFactory.getLogger(HiveHFileOutputFormat.class.getName());
private
@@ -95,6 +97,16 @@ public class HiveHFileOutputFormat extends
Properties tableProperties,
final Progressable progressable) throws IOException {
+ String hbaseTableName = jc.get(HBaseSerDe.HBASE_TABLE_NAME);
+ if (hbaseTableName == null) {
+ hbaseTableName = tableProperties.getProperty(hive_metastoreConstants.META_TABLE_NAME);
+ hbaseTableName = hbaseTableName.toLowerCase();
+ if (hbaseTableName.startsWith(HBaseStorageHandler.DEFAULT_PREFIX)) {
+ hbaseTableName = hbaseTableName.substring(HBaseStorageHandler.DEFAULT_PREFIX.length());
+ }
+ }
+ jc.set(OUTPUT_TABLE_NAME_CONF_KEY, hbaseTableName);
+
// Read configuration for the target path, first from jobconf, then from table properties
String hfilePath = getFamilyPath(jc, tableProperties);
if (hfilePath == null) {
http://git-wip-us.apache.org/repos/asf/hive/blob/1eddbc06/hbase-handler/src/test/queries/positive/hbase_bulk.q
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/queries/positive/hbase_bulk.q b/hbase-handler/src/test/queries/positive/hbase_bulk.q
index 5e0c14e..475aafc 100644
--- a/hbase-handler/src/test/queries/positive/hbase_bulk.q
+++ b/hbase-handler/src/test/queries/positive/hbase_bulk.q
@@ -9,7 +9,7 @@ create table hbsort(key string, val string, val2 string)
stored as
INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'
OUTPUTFORMAT 'org.apache.hadoop.hive.hbase.HiveHFileOutputFormat'
-TBLPROPERTIES ('hfile.family.path' = '/tmp/hbsort/cf','hbase.mapreduce.hfileoutputformat.table.name'='hbsort');
+TBLPROPERTIES ('hfile.family.path' = '/tmp/hbsort/cf');
-- this is a dummy table used for controlling how the input file
-- for TotalOrderPartitioner is created
http://git-wip-us.apache.org/repos/asf/hive/blob/1eddbc06/hbase-handler/src/test/queries/positive/hbase_handler_bulk.q
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/queries/positive/hbase_handler_bulk.q b/hbase-handler/src/test/queries/positive/hbase_handler_bulk.q
index 5ac4dc8..d02a61f 100644
--- a/hbase-handler/src/test/queries/positive/hbase_handler_bulk.q
+++ b/hbase-handler/src/test/queries/positive/hbase_handler_bulk.q
@@ -6,7 +6,7 @@ drop table if exists hb_target;
create table hb_target(key int, val string)
stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
with serdeproperties ('hbase.columns.mapping' = ':key,cf:val')
-tblproperties ('hbase.mapreduce.hfileoutputformat.table.name' = 'positive_hbase_handler_bulk');
+tblproperties ('hbase.table.name' = 'positive_hbase_handler_bulk');
set hive.hbase.generatehfiles=true;
set hfile.family.path=/tmp/hb_target/cf;
@@ -30,7 +30,7 @@ dfs -rmr /tmp/hb_target/cf;
create table hb_target(key int, val string)
stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
with serdeproperties ('hbase.columns.mapping' = ':key,cf:val')
-tblproperties ('hbase.mapreduce.hfileoutputformat.table.name' = 'positive_hbase_handler_bulk');
+tblproperties ('hbase.table.name' = 'positive_hbase_handler_bulk');
-- do it twice - regression test for HIVE-18607
http://git-wip-us.apache.org/repos/asf/hive/blob/1eddbc06/hbase-handler/src/test/results/positive/hbase_handler_bulk.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_handler_bulk.q.out b/hbase-handler/src/test/results/positive/hbase_handler_bulk.q.out
index cd8930f..de1c139 100644
--- a/hbase-handler/src/test/results/positive/hbase_handler_bulk.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_handler_bulk.q.out
@@ -5,14 +5,14 @@ POSTHOOK: type: DROPTABLE
PREHOOK: query: create table hb_target(key int, val string)
stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
with serdeproperties ('hbase.columns.mapping' = ':key,cf:val')
-tblproperties ('hbase.mapreduce.hfileoutputformat.table.name' = 'positive_hbase_handler_bulk')
+tblproperties ('hbase.table.name' = 'positive_hbase_handler_bulk')
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@hb_target
POSTHOOK: query: create table hb_target(key int, val string)
stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
with serdeproperties ('hbase.columns.mapping' = ':key,cf:val')
-tblproperties ('hbase.mapreduce.hfileoutputformat.table.name' = 'positive_hbase_handler_bulk')
+tblproperties ('hbase.table.name' = 'positive_hbase_handler_bulk')
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@hb_target
@@ -36,14 +36,14 @@ POSTHOOK: Output: default@hb_target
PREHOOK: query: create table hb_target(key int, val string)
stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
with serdeproperties ('hbase.columns.mapping' = ':key,cf:val')
-tblproperties ('hbase.mapreduce.hfileoutputformat.table.name' = 'positive_hbase_handler_bulk')
+tblproperties ('hbase.table.name' = 'positive_hbase_handler_bulk')
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@hb_target
POSTHOOK: query: create table hb_target(key int, val string)
stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
with serdeproperties ('hbase.columns.mapping' = ':key,cf:val')
-tblproperties ('hbase.mapreduce.hfileoutputformat.table.name' = 'positive_hbase_handler_bulk')
+tblproperties ('hbase.table.name' = 'positive_hbase_handler_bulk')
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@hb_target