You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by px...@apache.org on 2017/04/05 23:39:18 UTC
hive git commit: HIVE-16372: Enable DDL statement for non-native
tables (add/remove table properties) (Pengcheng Xiong,
reviewed by Thejas M Nair)
Repository: hive
Updated Branches:
refs/heads/master 674138a65 -> db24ce722
HIVE-16372: Enable DDL statement for non-native tables (add/remove table properties) (Pengcheng Xiong, reviewed by Thejas M Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/db24ce72
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/db24ce72
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/db24ce72
Branch: refs/heads/master
Commit: db24ce7224758a3c6bde4be2e796675ce4265e1f
Parents: 674138a
Author: Pengcheng Xiong <px...@hortonworks.com>
Authored: Wed Apr 5 16:39:08 2017 -0700
Committer: Pengcheng Xiong <px...@hortonworks.com>
Committed: Wed Apr 5 16:39:08 2017 -0700
----------------------------------------------------------------------
.../src/test/queries/negative/hbase_ddl.q | 9 +
.../src/test/queries/positive/hbase_ddl.q | 20 ++
.../src/test/results/negative/hbase_ddl.q.out | 29 +++
.../src/test/results/positive/hbase_ddl.q.out | 186 +++++++++++++++++++
.../org/apache/hadoop/hive/ql/ErrorMsg.java | 4 +-
.../hive/ql/parse/DDLSemanticAnalyzer.java | 2 +-
.../hadoop/hive/ql/plan/AlterTableDesc.java | 6 +
.../clientnegative/alter_non_native.q.out | 2 +-
8 files changed, 255 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/db24ce72/hbase-handler/src/test/queries/negative/hbase_ddl.q
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/queries/negative/hbase_ddl.q b/hbase-handler/src/test/queries/negative/hbase_ddl.q
new file mode 100644
index 0000000..2913bcd
--- /dev/null
+++ b/hbase-handler/src/test/queries/negative/hbase_ddl.q
@@ -0,0 +1,9 @@
+DROP TABLE hbase_table_1;
+CREATE TABLE hbase_table_1(key int comment 'It is a column key', value string comment 'It is the column string value')
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = "cf:string")
+TBLPROPERTIES ("hbase.table.name" = "hbase_table_0");
+
+DESCRIBE EXTENDED hbase_table_1;
+
+alter table hbase_table_1 change column key newkey string;
http://git-wip-us.apache.org/repos/asf/hive/blob/db24ce72/hbase-handler/src/test/queries/positive/hbase_ddl.q
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/queries/positive/hbase_ddl.q b/hbase-handler/src/test/queries/positive/hbase_ddl.q
new file mode 100644
index 0000000..a8bae75
--- /dev/null
+++ b/hbase-handler/src/test/queries/positive/hbase_ddl.q
@@ -0,0 +1,20 @@
+DROP TABLE hbase_table_1;
+CREATE TABLE hbase_table_1(key int comment 'It is a column key', value string comment 'It is the column string value')
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = "cf:string")
+TBLPROPERTIES ("hbase.table.name" = "hbase_table_0");
+
+DESCRIBE EXTENDED hbase_table_1;
+
+select * from hbase_table_1;
+
+EXPLAIN FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT * WHERE (key%2)=0;
+FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT * WHERE (key%2)=0;
+
+ALTER TABLE hbase_table_1 SET TBLPROPERTIES('hbase.mapred.output.outputtable'='kkk');
+
+desc formatted hbase_table_1;
+
+ALTER TABLE hbase_table_1 unset TBLPROPERTIES('hbase.mapred.output.outputtable');
+
+desc formatted hbase_table_1;
http://git-wip-us.apache.org/repos/asf/hive/blob/db24ce72/hbase-handler/src/test/results/negative/hbase_ddl.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/negative/hbase_ddl.q.out b/hbase-handler/src/test/results/negative/hbase_ddl.q.out
new file mode 100644
index 0000000..b5aad70
--- /dev/null
+++ b/hbase-handler/src/test/results/negative/hbase_ddl.q.out
@@ -0,0 +1,29 @@
+PREHOOK: query: DROP TABLE hbase_table_1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE hbase_table_1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE hbase_table_1(key int comment 'It is a column key', value string comment 'It is the column string value')
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = "cf:string")
+TBLPROPERTIES ("hbase.table.name" = "hbase_table_0")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hbase_table_1
+POSTHOOK: query: CREATE TABLE hbase_table_1(key int comment 'It is a column key', value string comment 'It is the column string value')
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = "cf:string")
+TBLPROPERTIES ("hbase.table.name" = "hbase_table_0")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hbase_table_1
+PREHOOK: query: DESCRIBE EXTENDED hbase_table_1
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@hbase_table_1
+POSTHOOK: query: DESCRIBE EXTENDED hbase_table_1
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@hbase_table_1
+key int It is a column key
+value string It is the column string value
+
+#### A masked pattern was here ####
+FAILED: SemanticException [Error 10134]: ALTER TABLE can only be used for [ADDPROPS, DROPPROPS] to a non-native table hbase_table_1
http://git-wip-us.apache.org/repos/asf/hive/blob/db24ce72/hbase-handler/src/test/results/positive/hbase_ddl.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_ddl.q.out b/hbase-handler/src/test/results/positive/hbase_ddl.q.out
new file mode 100644
index 0000000..8cb88ed
--- /dev/null
+++ b/hbase-handler/src/test/results/positive/hbase_ddl.q.out
@@ -0,0 +1,186 @@
+PREHOOK: query: DROP TABLE hbase_table_1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE hbase_table_1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE hbase_table_1(key int comment 'It is a column key', value string comment 'It is the column string value')
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = "cf:string")
+TBLPROPERTIES ("hbase.table.name" = "hbase_table_0")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hbase_table_1
+POSTHOOK: query: CREATE TABLE hbase_table_1(key int comment 'It is a column key', value string comment 'It is the column string value')
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ("hbase.columns.mapping" = "cf:string")
+TBLPROPERTIES ("hbase.table.name" = "hbase_table_0")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hbase_table_1
+PREHOOK: query: DESCRIBE EXTENDED hbase_table_1
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@hbase_table_1
+POSTHOOK: query: DESCRIBE EXTENDED hbase_table_1
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@hbase_table_1
+key int It is a column key
+value string It is the column string value
+
+#### A masked pattern was here ####
+PREHOOK: query: select * from hbase_table_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_table_1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from hbase_table_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_table_1
+#### A masked pattern was here ####
+PREHOOK: query: EXPLAIN FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT * WHERE (key%2)=0
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT * WHERE (key%2)=0
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+ Stage-2
+ Stage-1 is a root stage
+ Stage-3 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Alter Table Operator:
+ Alter Table
+ type: drop props
+ old name: default.hbase_table_1
+ properties:
+ COLUMN_STATS_ACCURATE
+
+ Stage: Stage-2
+ Insert operator:
+ Insert
+
+ Stage: Stage-1
+ Pre Insert operator:
+ Pre-Insert task
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((UDFToDouble(key) % 2.0) = 0.0) (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: UDFToInteger(key) (type: int), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.hbase.HiveHBaseTableInputFormat
+ output format: org.apache.hadoop.hive.hbase.HiveHBaseTableOutputFormat
+ serde: org.apache.hadoop.hive.hbase.HBaseSerDe
+ name: default.hbase_table_1
+
+PREHOOK: query: FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT * WHERE (key%2)=0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_1
+POSTHOOK: query: FROM src INSERT OVERWRITE TABLE hbase_table_1 SELECT * WHERE (key%2)=0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_1
+PREHOOK: query: ALTER TABLE hbase_table_1 SET TBLPROPERTIES('hbase.mapred.output.outputtable'='kkk')
+PREHOOK: type: ALTERTABLE_PROPERTIES
+PREHOOK: Input: default@hbase_table_1
+PREHOOK: Output: default@hbase_table_1
+POSTHOOK: query: ALTER TABLE hbase_table_1 SET TBLPROPERTIES('hbase.mapred.output.outputtable'='kkk')
+POSTHOOK: type: ALTERTABLE_PROPERTIES
+POSTHOOK: Input: default@hbase_table_1
+POSTHOOK: Output: default@hbase_table_1
+PREHOOK: query: desc formatted hbase_table_1
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@hbase_table_1
+POSTHOOK: query: desc formatted hbase_table_1
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@hbase_table_1
+# col_name data_type comment
+
+key int It is a column key
+value string It is the column string value
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ hbase.mapred.output.outputtable kkk
+ hbase.table.name hbase_table_0
+#### A masked pattern was here ####
+ numFiles 0
+ numRows 0
+ rawDataSize 0
+ storage_handler org.apache.hadoop.hive.hbase.HBaseStorageHandler
+ totalSize 0
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.hbase.HBaseSerDe
+InputFormat: null
+OutputFormat: null
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ hbase.columns.mapping cf:string
+ serialization.format 1
+PREHOOK: query: ALTER TABLE hbase_table_1 unset TBLPROPERTIES('hbase.mapred.output.outputtable')
+PREHOOK: type: ALTERTABLE_PROPERTIES
+PREHOOK: Input: default@hbase_table_1
+PREHOOK: Output: default@hbase_table_1
+POSTHOOK: query: ALTER TABLE hbase_table_1 unset TBLPROPERTIES('hbase.mapred.output.outputtable')
+POSTHOOK: type: ALTERTABLE_PROPERTIES
+POSTHOOK: Input: default@hbase_table_1
+POSTHOOK: Output: default@hbase_table_1
+PREHOOK: query: desc formatted hbase_table_1
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@hbase_table_1
+POSTHOOK: query: desc formatted hbase_table_1
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@hbase_table_1
+# col_name data_type comment
+
+key int It is a column key
+value string It is the column string value
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ hbase.table.name hbase_table_0
+#### A masked pattern was here ####
+ numFiles 0
+ numRows 0
+ rawDataSize 0
+ storage_handler org.apache.hadoop.hive.hbase.HBaseStorageHandler
+ totalSize 0
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.hbase.HBaseSerDe
+InputFormat: null
+OutputFormat: null
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ hbase.columns.mapping cf:string
+ serialization.format 1
http://git-wip-us.apache.org/repos/asf/hive/blob/db24ce72/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index 6a43385..746c199 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql;
import java.text.MessageFormat;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
@@ -27,6 +28,7 @@ import java.util.regex.Pattern;
import org.antlr.runtime.tree.Tree;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.ASTNodeOrigin;
+import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
/**
* List of all error messages.
@@ -217,7 +219,7 @@ public enum ErrorMsg {
ALTER_COMMAND_FOR_VIEWS(10131, "To alter a view you need to use the ALTER VIEW command."),
ALTER_COMMAND_FOR_TABLES(10132, "To alter a base table you need to use the ALTER TABLE command."),
ALTER_VIEW_DISALLOWED_OP(10133, "Cannot use this form of ALTER on a view"),
- ALTER_TABLE_NON_NATIVE(10134, "ALTER TABLE cannot be used for a non-native table"),
+ ALTER_TABLE_NON_NATIVE(10134, "ALTER TABLE can only be used for " + AlterTableTypes.nonNativeTableAllowedTypes + " to a non-native table "),
SORTMERGE_MAPJOIN_FAILED(10135,
"Sort merge bucketed join could not be performed. " +
"If you really want to perform the operation, either set " +
http://git-wip-us.apache.org/repos/asf/hive/blob/db24ce72/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index 6e72d07..0cf9205 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@ -1350,7 +1350,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
throw new SemanticException(ErrorMsg.ALTER_COMMAND_FOR_TABLES.getMsg());
}
}
- if (tbl.isNonNative()) {
+ if (tbl.isNonNative() && !AlterTableTypes.nonNativeTableAllowedTypes.contains(op)) {
throw new SemanticException(ErrorMsg.ALTER_TABLE_NON_NATIVE.getMsg(tbl.getTableName()));
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/db24ce72/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
index b83c16d..d5a6679 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
@@ -29,8 +29,11 @@ import org.apache.hadoop.hive.ql.parse.ParseUtils;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
+import com.google.common.collect.ImmutableList;
+
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -64,6 +67,9 @@ public class AlterTableDesc extends DDLDesc implements Serializable {
private final String name;
private AlterTableTypes(String name) { this.name = name; }
public String getName() { return name; }
+
+ public static final List<AlterTableTypes> nonNativeTableAllowedTypes =
+ ImmutableList.of(ADDPROPS, DROPPROPS);
}
public static enum ProtectModeType {
http://git-wip-us.apache.org/repos/asf/hive/blob/db24ce72/ql/src/test/results/clientnegative/alter_non_native.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/alter_non_native.q.out b/ql/src/test/results/clientnegative/alter_non_native.q.out
index 64c4cee..0a32067 100644
--- a/ql/src/test/results/clientnegative/alter_non_native.q.out
+++ b/ql/src/test/results/clientnegative/alter_non_native.q.out
@@ -8,4 +8,4 @@ STORED BY 'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler'
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@non_native1
-FAILED: SemanticException [Error 10134]: ALTER TABLE cannot be used for a non-native table non_native1
+FAILED: SemanticException [Error 10134]: ALTER TABLE can only be used for [ADDPROPS, DROPPROPS] to a non-native table non_native1