You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/06/05 19:23:06 UTC
[6/6] hive git commit: HIVE-19643 : MM table conversion doesn't need
full ACID structure checks (Sergey Shelukhin, reviewed by Eugene Koifman)
HIVE-19643 : MM table conversion doesn't need full ACID structure checks (Sergey Shelukhin, reviewed by Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ba621257
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ba621257
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ba621257
Branch: refs/heads/branch-3
Commit: ba621257c4878f78632c6fe9f1f7bbe2fb304e8f
Parents: b6090fa
Author: sergey <se...@apache.org>
Authored: Fri Jun 1 12:53:01 2018 -0700
Committer: sergey <se...@apache.org>
Committed: Tue Jun 5 12:17:46 2018 -0700
----------------------------------------------------------------------
.../queries/clientpositive/mm_conversions.q | 11 ++++
.../clientpositive/llap/mm_conversions.q.out | 62 ++++++++++++++++++++
.../TransactionalValidationListener.java | 25 ++++----
3 files changed, 87 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ba621257/ql/src/test/queries/clientpositive/mm_conversions.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/mm_conversions.q b/ql/src/test/queries/clientpositive/mm_conversions.q
index ae45c18..89beede 100644
--- a/ql/src/test/queries/clientpositive/mm_conversions.q
+++ b/ql/src/test/queries/clientpositive/mm_conversions.q
@@ -60,6 +60,17 @@ insert into table part_to_mm partition(key_mm='457') select key from intermediat
select * from part_to_mm s3 order by key, key_mm;
drop table part_to_mm;
+
+drop table load_to_mm;
+create table load_to_mm (key string, value string) tblproperties("transactional"="false");
+load data local inpath '../../data/files/kv1.txt' into table load_to_mm;
+load data local inpath '../../data/files/kv1.txt' into table load_to_mm;
+select count(*) from load_to_mm s1;
+alter table load_to_mm set tblproperties("transactional"="true", "transactional_properties"="insert_only");
+select count(*) from load_to_mm s2;
+drop table load_to_mm;
+
+
set hive.mm.allow.originals=false;
drop table simple_to_mm_text;
http://git-wip-us.apache.org/repos/asf/hive/blob/ba621257/ql/src/test/results/clientpositive/llap/mm_conversions.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/mm_conversions.q.out b/ql/src/test/results/clientpositive/llap/mm_conversions.q.out
index 8a9036a..618aa3c 100644
--- a/ql/src/test/results/clientpositive/llap/mm_conversions.q.out
+++ b/ql/src/test/results/clientpositive/llap/mm_conversions.q.out
@@ -460,6 +460,68 @@ POSTHOOK: query: drop table part_to_mm
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@part_to_mm
POSTHOOK: Output: default@part_to_mm
+PREHOOK: query: drop table load_to_mm
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table load_to_mm
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table load_to_mm (key string, value string) tblproperties("transactional"="false")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@load_to_mm
+POSTHOOK: query: create table load_to_mm (key string, value string) tblproperties("transactional"="false")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@load_to_mm
+PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table load_to_mm
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load_to_mm
+POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table load_to_mm
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load_to_mm
+PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table load_to_mm
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load_to_mm
+POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table load_to_mm
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load_to_mm
+PREHOOK: query: select count(*) from load_to_mm s1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@load_to_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from load_to_mm s1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@load_to_mm
+#### A masked pattern was here ####
+1000
+PREHOOK: query: alter table load_to_mm set tblproperties("transactional"="true", "transactional_properties"="insert_only")
+PREHOOK: type: ALTERTABLE_PROPERTIES
+PREHOOK: Input: default@load_to_mm
+PREHOOK: Output: default@load_to_mm
+POSTHOOK: query: alter table load_to_mm set tblproperties("transactional"="true", "transactional_properties"="insert_only")
+POSTHOOK: type: ALTERTABLE_PROPERTIES
+POSTHOOK: Input: default@load_to_mm
+POSTHOOK: Output: default@load_to_mm
+PREHOOK: query: select count(*) from load_to_mm s2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@load_to_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from load_to_mm s2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@load_to_mm
+#### A masked pattern was here ####
+1000
+PREHOOK: query: drop table load_to_mm
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@load_to_mm
+PREHOOK: Output: default@load_to_mm
+POSTHOOK: query: drop table load_to_mm
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@load_to_mm
+POSTHOOK: Output: default@load_to_mm
PREHOOK: query: drop table simple_to_mm_text
PREHOOK: type: DROPTABLE
POSTHOOK: query: drop table simple_to_mm_text
http://git-wip-us.apache.org/repos/asf/hive/blob/ba621257/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
index 42d0248..c3d99c3 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/TransactionalValidationListener.java
@@ -132,24 +132,26 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
}
if ("true".equalsIgnoreCase(transactionalValue) && !"true".equalsIgnoreCase(oldTransactionalValue)) {
if(!isTransactionalPropertiesPresent) {
- normazlieTransactionalPropertyDefault(newTable);
+ normalizeTransactionalPropertyDefault(newTable);
isTransactionalPropertiesPresent = true;
transactionalPropertiesValue = DEFAULT_TRANSACTIONAL_PROPERTY;
}
- //only need to check conformance if alter table enabled acid
- if (!conformToAcid(newTable)) {
- // INSERT_ONLY tables don't have to conform to ACID requirement like ORC or bucketing
- if (transactionalPropertiesValue == null || !"insert_only".equalsIgnoreCase(transactionalPropertiesValue)) {
- throw new MetaException("The table must be stored using an ACID compliant format (such as ORC): "
- + Warehouse.getQualifiedName(newTable));
- }
+ // We only need to check conformance if alter table enabled acid.
+ // INSERT_ONLY tables don't have to conform to ACID requirement like ORC or bucketing.
+ boolean isFullAcid = transactionalPropertiesValue == null
+ || !"insert_only".equalsIgnoreCase(transactionalPropertiesValue);
+ if (isFullAcid && !conformToAcid(newTable)) {
+ throw new MetaException("The table must be stored using an ACID compliant "
+ + "format (such as ORC): " + Warehouse.getQualifiedName(newTable));
}
if (newTable.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) {
throw new MetaException(Warehouse.getQualifiedName(newTable) +
" cannot be declared transactional because it's an external table");
}
- validateTableStructure(context.getHandler(), newTable);
+ if (isFullAcid) {
+ validateTableStructure(context.getHandler(), newTable);
+ }
hasValidTransactionalValue = true;
}
@@ -189,6 +191,7 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
}
checkSorted(newTable);
}
+
private void checkSorted(Table newTable) throws MetaException {
if(!TxnUtils.isAcidTable(newTable)) {
return;
@@ -309,7 +312,7 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
// normalize prop name
parameters.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, Boolean.TRUE.toString());
if(transactionalProperties == null) {
- normazlieTransactionalPropertyDefault(newTable);
+ normalizeTransactionalPropertyDefault(newTable);
}
initializeTransactionalProperties(newTable);
checkSorted(newTable);
@@ -325,7 +328,7 @@ public final class TransactionalValidationListener extends MetaStorePreEventList
* transactional_properties should take on the default value. Easier to make this explicit in
* table definition than keep checking everywhere if it's set or not.
*/
- private void normazlieTransactionalPropertyDefault(Table table) {
+ private void normalizeTransactionalPropertyDefault(Table table) {
table.getParameters().put(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES,
DEFAULT_TRANSACTIONAL_PROPERTY);