You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kg...@apache.org on 2019/06/17 07:59:56 UTC
[hive] 01/02: HIVE-21830: Break up DDLTask - extract rest of the
Alter Table operations (Miklos Gergely via Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 541fe86a07ea749a5b8e663d79e55dcb39225039
Author: Miklos Gergely <mg...@hortonworks.com>
AuthorDate: Mon Jun 17 09:57:22 2019 +0200
HIVE-21830: Break up DDLTask - extract rest of the Alter Table operations (Miklos Gergely via Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
---
.../test/results/positive/accumulo_queries.q.out | 20 +-
.../accumulo_single_sourced_multi_insert.q.out | 10 +-
.../hadoop/hive/druid/DruidStorageHandler.java | 3 +-
.../src/test/results/negative/hbase_ddl.q.out | 2 +-
.../src/test/results/positive/hbase_ddl.q.out | 10 +-
.../src/test/results/positive/hbase_queries.q.out | 20 +-
.../hbase_single_sourced_multi_insert.q.out | 10 +-
.../src/test/results/positive/hbasestats.q.out | 10 +-
.../java/org/apache/hadoop/hive/ql/ErrorMsg.java | 5 +-
.../hive/ql/ddl/table/AbstractAlterTableDesc.java | 16 +-
.../ql/ddl/table/AbstractAlterTableOperation.java | 5 +-
.../AbstractAlterTableWithConstraintsDesc.java | 3 +-
.../hadoop/hive/ql/ddl/table/AlterTableType.java | 85 ++
.../ddl/table/column/AlterTableAddColumnsDesc.java | 4 +-
.../table/column/AlterTableChangeColumnDesc.java | 4 +-
.../table/column/AlterTableReplaceColumnsDesc.java | 4 +-
.../table/column/AlterTableUpdateColumnsDesc.java | 4 +-
.../constaint/AlterTableAddConstraintDesc.java | 4 +-
.../AlterTableRenameDesc.java} | 29 +-
.../ddl/table/misc/AlterTableRenameOperation.java | 56 +
.../AlterTableSetOwnerDesc.java} | 30 +-
.../table/misc/AlterTableSetOwnerOperation.java | 45 +
.../table/misc/AlterTableSetPropertiesDesc.java | 67 ++
.../misc/AlterTableSetPropertiesOperation.java | 159 +++
.../AlterTableTouchDesc.java} | 35 +-
.../ddl/table/misc/AlterTableTouchOperation.java | 71 ++
.../table/misc/AlterTableUnsetPropertiesDesc.java | 63 +
.../misc/AlterTableUnsetPropertiesOperation.java | 71 ++
...tSortedDesc.java => AlterTableArchiveDesc.java} | 33 +-
.../table/storage/AlterTableArchiveOperation.java | 318 +++++
.../ddl/table/storage/AlterTableArchiveUtils.java | 123 ++
.../table/storage/AlterTableClusteredByDesc.java | 4 +-
.../ddl/table/storage/AlterTableCompactDesc.java | 78 ++
.../table/storage/AlterTableCompactOperation.java | 136 +++
.../table/storage/AlterTableConcatenateDesc.java} | 39 +-
.../storage/AlterTableConcatenateOperation.java | 118 ++
.../table/storage/AlterTableIntoBucketsDesc.java | 4 +-
.../table/storage/AlterTableNotClusteredDesc.java | 4 +-
.../ddl/table/storage/AlterTableNotSkewedDesc.java | 4 +-
.../ddl/table/storage/AlterTableNotSortedDesc.java | 4 +-
.../table/storage/AlterTableSetFileFormatDesc.java | 4 +-
.../table/storage/AlterTableSetLocationDesc.java | 4 +-
.../ddl/table/storage/AlterTableSetSerdeDesc.java | 4 +-
.../table/storage/AlterTableSetSerdePropsDesc.java | 4 +-
.../storage/AlterTableSetSkewedLocationDesc.java | 4 +-
.../ddl/table/storage/AlterTableSkewedByDesc.java | 4 +-
...ortedDesc.java => AlterTableUnarchiveDesc.java} | 33 +-
.../storage/AlterTableUnarchiveOperation.java | 292 +++++
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 1244 --------------------
.../org/apache/hadoop/hive/ql/exec/ExportTask.java | 2 +-
.../apache/hadoop/hive/ql/exec/ReplCopyTask.java | 2 +-
.../java/org/apache/hadoop/hive/ql/exec/Task.java | 2 +-
.../ql/exec/repl/ExternalTableCopyTaskBuilder.java | 2 +-
.../hadoop/hive/ql/exec/repl/ReplDumpTask.java | 2 +-
.../hadoop/hive/ql/exec/repl/ReplLoadTask.java | 2 +-
.../incremental/IncrementalLoadTasksBuilder.java | 12 +-
.../hadoop/hive/ql/exec/repl/util/ReplUtils.java | 17 +-
.../apache/hadoop/hive/ql/hooks/WriteEntity.java | 8 +-
.../org/apache/hadoop/hive/ql/io/HdfsUtils.java | 11 +
.../hive/ql/parse/AcidExportSemanticAnalyzer.java | 12 +-
.../hadoop/hive/ql/parse/DDLSemanticAnalyzer.java | 298 ++---
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 13 +-
.../repl/load/message/RenameTableHandler.java | 11 +-
.../apache/hadoop/hive/ql/plan/AlterTableDesc.java | 406 -------
.../hadoop/hive/ql/plan/AlterTableSimpleDesc.java | 119 --
.../org/apache/hadoop/hive/ql/plan/DDLWork.java | 54 -
.../apache/hadoop/hive/ql/parse/TestQBCompact.java | 18 +-
.../acid_vectorization_original_tez.q | 1 +
ql/src/test/queries/clientpositive/archive_multi.q | 2 +
.../queries/clientpositive/dbtxnmgr_compact2.q | 1 +
.../queries/clientpositive/set_tblproperties.q | 8 +
.../test/queries/clientpositive/table_set_owner.q | 4 +
ql/src/test/queries/clientpositive/touch.q | 2 +
.../clientnegative/alter_external_acid.q.out | 2 +-
.../results/clientnegative/alter_non_native.q.out | 2 +-
.../clientnegative/alter_table_wrong_db.q.out | 2 +-
ql/src/test/results/clientnegative/archive1.q.out | 2 +-
ql/src/test/results/clientnegative/archive2.q.out | 2 +-
.../results/clientnegative/archive_multi1.q.out | 2 +-
.../results/clientnegative/archive_multi2.q.out | 2 +-
.../results/clientnegative/archive_multi3.q.out | 2 +-
.../results/clientnegative/archive_multi4.q.out | 2 +-
.../results/clientnegative/archive_multi5.q.out | 2 +-
.../results/clientnegative/archive_multi6.q.out | 2 +-
.../clientnegative/compact_non_acid_table.q.out | 2 +-
.../test/results/clientnegative/mm_convert.q.out | 2 +-
.../clientnegative/strict_managed_tables2.q.out | 2 +-
.../clientnegative/strict_managed_tables3.q.out | 2 +-
.../results/clientnegative/temp_table_rename.q.out | 2 +-
ql/src/test/results/clientnegative/touch1.q.out | 2 +-
.../clientpositive/alter_rename_table.q.out | 16 +-
.../results/clientpositive/archive_multi.q.out | 40 +
.../results/clientpositive/dbtxnmgr_compact2.q.out | 15 +
.../druid/druidmini_dynamic_partition.q.out | 20 +-
.../clientpositive/druid/druidmini_mv.q.out | 10 +-
.../encrypted/encryption_move_tbl.q.out | 2 +-
ql/src/test/results/clientpositive/input3.q.out | 8 +-
.../results/clientpositive/llap/orc_merge10.q.out | 5 +
.../results/clientpositive/llap/orc_merge5.q.out | 2 +
.../results/clientpositive/llap/orc_merge6.q.out | 5 +
.../results/clientpositive/llap/orc_merge7.q.out | 4 +
.../clientpositive/llap/orc_merge_incompat2.q.out | 4 +
.../test/results/clientpositive/orc_merge10.q.out | 5 +
.../test/results/clientpositive/orc_merge5.q.out | 2 +
.../test/results/clientpositive/orc_merge6.q.out | 5 +
.../clientpositive/orc_merge_incompat2.q.out | 4 +
.../results/clientpositive/set_tblproperties.q.out | 102 ++
.../results/clientpositive/spark/orc_merge5.q.out | 2 +
.../results/clientpositive/spark/orc_merge6.q.out | 5 +
.../results/clientpositive/spark/orc_merge7.q.out | 4 +
.../clientpositive/spark/orc_merge_incompat2.q.out | 4 +
.../results/clientpositive/table_set_owner.q.out | 28 +
.../results/clientpositive/table_storage.q.out | 2 +-
.../tez/acid_vectorization_original_tez.q.out | 17 +
.../clientpositive/tez/explainanalyze_3.q.out | 3 +-
.../results/clientpositive/tez/explainuser_3.q.out | 3 +-
ql/src/test/results/clientpositive/touch.q.out | 35 +
.../apache/hadoop/hive/metastore/HiveMetaHook.java | 4 +-
.../hadoop/hive/metastore/HiveAlterHandler.java | 6 +-
119 files changed, 2365 insertions(+), 2343 deletions(-)
diff --git a/accumulo-handler/src/test/results/positive/accumulo_queries.q.out b/accumulo-handler/src/test/results/positive/accumulo_queries.q.out
index c5379c7..32a6e44 100644
--- a/accumulo-handler/src/test/results/positive/accumulo_queries.q.out
+++ b/accumulo-handler/src/test/results/positive/accumulo_queries.q.out
@@ -51,12 +51,10 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
- Alter Table Operator:
- Alter Table
- type: drop props
- old name: default.accumulo_table_1
- properties:
- COLUMN_STATS_ACCURATE
+ Unset Properties
+ properties:
+ COLUMN_STATS_ACCURATE
+ table name: default.accumulo_table_1
Stage: Stage-1
Pre-Insert task
@@ -534,12 +532,10 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
- Alter Table Operator:
- Alter Table
- type: drop props
- old name: default.accumulo_table_3
- properties:
- COLUMN_STATS_ACCURATE
+ Unset Properties
+ properties:
+ COLUMN_STATS_ACCURATE
+ table name: default.accumulo_table_3
Stage: Stage-1
Pre-Insert task
diff --git a/accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out b/accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out
index 1a5dde0..12343d5 100644
--- a/accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out
+++ b/accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out
@@ -54,12 +54,10 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
- Alter Table Operator:
- Alter Table
- type: drop props
- old name: default.src_x2
- properties:
- COLUMN_STATS_ACCURATE
+ Unset Properties
+ properties:
+ COLUMN_STATS_ACCURATE
+ table name: default.src_x2
Stage: Stage-4
Stats Work
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
index 254d0a3..a14837d 100644
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
+++ b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
@@ -137,7 +137,8 @@ import static org.apache.hadoop.hive.druid.DruidStorageHandlerUtils.JSON_MAPPER;
private static final HttpClient HTTP_CLIENT;
- private static final List<String> ALLOWED_ALTER_TYPES = ImmutableList.of("ADDPROPS", "DROPPROPS", "ADD_COLUMNS");
+ private static final List<String> ALLOWED_ALTER_TYPES =
+ ImmutableList.of("ADDPROPS", "DROPPROPS", "ADDCOLS");
static {
final Lifecycle lifecycle = new Lifecycle();
diff --git a/hbase-handler/src/test/results/negative/hbase_ddl.q.out b/hbase-handler/src/test/results/negative/hbase_ddl.q.out
index 4646def..e4c146b 100644
--- a/hbase-handler/src/test/results/negative/hbase_ddl.q.out
+++ b/hbase-handler/src/test/results/negative/hbase_ddl.q.out
@@ -26,4 +26,4 @@ key int It is a column key
value string It is the column string value
#### A masked pattern was here ####
-FAILED: SemanticException [Error 10134]: ALTER TABLE can only be used for [ADDPROPS, DROPPROPS, ADD_COLUMNS] to a non-native table hbase_table_1
+FAILED: SemanticException [Error 10134]: ALTER TABLE can only be used for [ADDPROPS, DROPPROPS, ADDCOLS] to a non-native table hbase_table_1
diff --git a/hbase-handler/src/test/results/positive/hbase_ddl.q.out b/hbase-handler/src/test/results/positive/hbase_ddl.q.out
index 779ca4d..db3ef9e 100644
--- a/hbase-handler/src/test/results/positive/hbase_ddl.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_ddl.q.out
@@ -49,12 +49,10 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
- Alter Table Operator:
- Alter Table
- type: drop props
- old name: default.hbase_table_1
- properties:
- COLUMN_STATS_ACCURATE
+ Unset Properties
+ properties:
+ COLUMN_STATS_ACCURATE
+ table name: default.hbase_table_1
Stage: Stage-1
Pre-Insert task
diff --git a/hbase-handler/src/test/results/positive/hbase_queries.q.out b/hbase-handler/src/test/results/positive/hbase_queries.q.out
index adf8864..0c21d6d 100644
--- a/hbase-handler/src/test/results/positive/hbase_queries.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_queries.q.out
@@ -49,12 +49,10 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
- Alter Table Operator:
- Alter Table
- type: drop props
- old name: default.hbase_table_1
- properties:
- COLUMN_STATS_ACCURATE
+ Unset Properties
+ properties:
+ COLUMN_STATS_ACCURATE
+ table name: default.hbase_table_1
Stage: Stage-1
Pre-Insert task
@@ -534,12 +532,10 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
- Alter Table Operator:
- Alter Table
- type: drop props
- old name: default.hbase_table_3
- properties:
- COLUMN_STATS_ACCURATE
+ Unset Properties
+ properties:
+ COLUMN_STATS_ACCURATE
+ table name: default.hbase_table_3
Stage: Stage-1
Pre-Insert task
diff --git a/hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out b/hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out
index d474b4d..e3d0b9d 100644
--- a/hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out
@@ -54,12 +54,10 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
- Alter Table Operator:
- Alter Table
- type: drop props
- old name: default.src_x2
- properties:
- COLUMN_STATS_ACCURATE
+ Unset Properties
+ properties:
+ COLUMN_STATS_ACCURATE
+ table name: default.src_x2
Stage: Stage-4
Stats Work
diff --git a/hbase-handler/src/test/results/positive/hbasestats.q.out b/hbase-handler/src/test/results/positive/hbasestats.q.out
index 367b479..7b1f152 100644
--- a/hbase-handler/src/test/results/positive/hbasestats.q.out
+++ b/hbase-handler/src/test/results/positive/hbasestats.q.out
@@ -76,12 +76,10 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
- Alter Table Operator:
- Alter Table
- type: drop props
- old name: default.users
- properties:
- COLUMN_STATS_ACCURATE
+ Unset Properties
+ properties:
+ COLUMN_STATS_ACCURATE
+ table name: default.users
Stage: Stage-1
Pre-Insert task
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index bcfbf5b..07b139e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -23,9 +23,9 @@ import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.UnresolvedPathException;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.ASTNodeOrigin;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.security.AccessControlException;
import java.io.FileNotFoundException;
@@ -214,7 +214,8 @@ public enum ErrorMsg {
ALTER_COMMAND_FOR_VIEWS(10131, "To alter a view you need to use the ALTER VIEW command."),
ALTER_COMMAND_FOR_TABLES(10132, "To alter a base table you need to use the ALTER TABLE command."),
ALTER_VIEW_DISALLOWED_OP(10133, "Cannot use this form of ALTER on a view"),
- ALTER_TABLE_NON_NATIVE(10134, "ALTER TABLE can only be used for " + AlterTableTypes.nonNativeTableAllowedTypes + " to a non-native table "),
+ ALTER_TABLE_NON_NATIVE(10134, "ALTER TABLE can only be used for " + AlterTableType.NON_NATIVE_TABLE_ALLOWED +
+ " to a non-native table "),
SORTMERGE_MAPJOIN_FAILED(10135,
"Sort merge bucketed join could not be performed. " +
"If you really want to perform the operation, either set " +
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableDesc.java
index 432779b..0e1c817 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableDesc.java
@@ -21,12 +21,12 @@ package org.apache.hadoop.hive.ql.ddl.table;
import java.io.Serializable;
import java.util.Map;
+import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
import org.apache.hadoop.hive.ql.ddl.DDLDesc;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.Explain;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.plan.DDLDesc.DDLDescWithWriteId;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
@@ -36,7 +36,7 @@ import org.apache.hadoop.hive.ql.plan.Explain.Level;
public abstract class AbstractAlterTableDesc implements DDLDesc, DDLDescWithWriteId, Serializable {
private static final long serialVersionUID = 1L;
- private final AlterTableTypes type;
+ private final AlterTableType type;
private final String tableName;
private final Map<String, String> partitionSpec;
private final ReplicationSpec replicationSpec;
@@ -46,11 +46,11 @@ public abstract class AbstractAlterTableDesc implements DDLDesc, DDLDescWithWrit
private Long writeId;
- public AbstractAlterTableDesc(AlterTableTypes type, String tableName, Map<String, String> partitionSpec,
+ public AbstractAlterTableDesc(AlterTableType type, String tableName, Map<String, String> partitionSpec,
ReplicationSpec replicationSpec, boolean isCascade, boolean expectView, Map<String, String> props)
throws SemanticException {
this.type = type;
- this.tableName = String.join(".", Utilities.getDbTableName(tableName));
+ this.tableName = tableName.contains(".") ? tableName : String.join(".", Utilities.getDbTableName(tableName));
this.partitionSpec = partitionSpec;
this.replicationSpec = replicationSpec;
this.isCascade = isCascade;
@@ -58,7 +58,7 @@ public abstract class AbstractAlterTableDesc implements DDLDesc, DDLDescWithWrit
this.props = props;
}
- public AlterTableTypes getType() {
+ public AlterTableType getType() {
return type;
}
@@ -86,11 +86,15 @@ public abstract class AbstractAlterTableDesc implements DDLDesc, DDLDescWithWrit
return expectView;
}
- @Explain(displayName = "props", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+ @Explain(displayName = "properties", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public Map<String, String> getProps() {
return props;
}
+ public EnvironmentContext getEnvironmentContext() {
+ return null;
+ };
+
@Override
public String getFullTableName() {
return tableName;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableOperation.java
index baf98da..e771033 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableOperation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableOperation.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.session.SessionState;
/**
@@ -74,7 +73,7 @@ public abstract class AbstractAlterTableOperation extends DDLOperation {
// Don't change the table object returned by the metastore, as we'll mess with it's caches.
Table table = oldTable.copy();
- environmentContext = initializeEnvironmentContext(null);
+ environmentContext = initializeEnvironmentContext(desc.getEnvironmentContext());
if (partitions == null) {
doAlteration(table, null);
@@ -157,7 +156,7 @@ public abstract class AbstractAlterTableOperation extends DDLOperation {
// Note: this is necessary for UPDATE_STATISTICS command, that operates via ADDPROPS (why?).
// For any other updates, we don't want to do txn check on partitions when altering table.
boolean isTxn = false;
- if (alterTable.getPartitionSpec() != null && alterTable.getType() == AlterTableTypes.ADDPROPS) {
+ if (alterTable.getPartitionSpec() != null && alterTable.getType() == AlterTableType.ADDPROPS) {
// ADDPROPS is used to add replication properties like repl.last.id, which isn't
// transactional change. In case of replication check for transactional properties
// explicitly.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableWithConstraintsDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableWithConstraintsDesc.java
index 9babf2a..9ea0b8d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableWithConstraintsDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableWithConstraintsDesc.java
@@ -23,7 +23,6 @@ import java.util.Map;
import org.apache.hadoop.hive.ql.ddl.table.constaint.Constraints;
import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
/**
* Abstract ancestor of all ALTER TABLE descriptors that are handled by the AlterTableWithWriteIdOperations framework
@@ -34,7 +33,7 @@ public abstract class AbstractAlterTableWithConstraintsDesc extends AbstractAlte
private final Constraints constraints;
- public AbstractAlterTableWithConstraintsDesc(AlterTableTypes type, String tableName,
+ public AbstractAlterTableWithConstraintsDesc(AlterTableType type, String tableName,
Map<String, String> partitionSpec, ReplicationSpec replicationSpec, boolean isCascade, boolean expectView,
Map<String, String> props, Constraints constraints) throws SemanticException {
super(type, tableName, partitionSpec, replicationSpec, isCascade, expectView, props);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java
new file mode 100644
index 0000000..a7894e2
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.table;
+
+import java.util.List;
+import java.util.Set;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+
+/**
+ * Enumeration of alter table command types.
+ */
+public enum AlterTableType {
+ // column
+ ADDCOLS("add columns"),
+ REPLACE_COLUMNS("replace columns"),
+ RENAME_COLUMN("rename column"),
+ UPDATE_COLUMNS("update columns"),
+ // partition
+ ADDPARTITION("add partition"),
+ DROPPARTITION("drop partition"),
+ RENAMEPARTITION("rename partition"), // Note: used in RenamePartitionDesc, not here.
+ ALTERPARTITION("alter partition"), // Note: this is never used in AlterTableDesc.
+ // constraint
+ ADD_CONSTRAINT("add constraint"),
+ DROP_CONSTRAINT("drop constraint"),
+ // storage
+ SET_SERDE("set serde"),
+ SET_SERDE_PROPS("set serde props"),
+ SET_FILE_FORMAT("add fileformat"),
+ CLUSTERED_BY("clustered by"),
+ NOT_SORTED("not sorted"),
+ NOT_CLUSTERED("not clustered"),
+ ALTERLOCATION("set location"),
+ SKEWED_BY("skewed by"),
+ NOT_SKEWED("not skewed"),
+ SET_SKEWED_LOCATION("alter skew location"),
+ INTO_BUCKETS("alter bucket number"),
+ // misc
+ ADDPROPS("set properties"),
+ DROPPROPS("unset properties"),
+ TOUCH("touch"),
+ RENAME("rename"),
+ OWNER("set owner"),
+ ARCHIVE("archieve"),
+ UNARCHIVE("unarchieve"),
+ COMPACT("compact"),
+ TRUNCATE("truncate"),
+ MERGEFILES("merge files"),
+ UPDATESTATS("update stats"); // Note: used in ColumnStatsUpdateWork, not here.
+
+ private final String name;
+
+ AlterTableType(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public static final List<AlterTableType> NON_NATIVE_TABLE_ALLOWED =
+ ImmutableList.of(ADDPROPS, DROPPROPS, ADDCOLS);
+
+ public static final Set<AlterTableType> SUPPORT_PARTIAL_PARTITION_SPEC =
+ ImmutableSet.of(ADDCOLS, REPLACE_COLUMNS, RENAME_COLUMN, ADDPROPS, DROPPROPS, SET_SERDE,
+ SET_SERDE_PROPS, SET_FILE_FORMAT);
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableAddColumnsDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableAddColumnsDesc.java
index e40ba18..d08276e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableAddColumnsDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableAddColumnsDesc.java
@@ -23,9 +23,9 @@ import java.util.Map;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.ddl.DDLTask2;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
@@ -44,7 +44,7 @@ public class AlterTableAddColumnsDesc extends AbstractAlterTableDesc {
public AlterTableAddColumnsDesc(String tableName, Map<String, String> partitionSpec, boolean isCascade,
List<FieldSchema> newColumns) throws SemanticException {
- super(AlterTableTypes.ADD_COLUMNS, tableName, partitionSpec, null, isCascade, false, null);
+ super(AlterTableType.ADDCOLS, tableName, partitionSpec, null, isCascade, false, null);
this.newColumns = newColumns;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableChangeColumnDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableChangeColumnDesc.java
index ce3b97e..a619acc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableChangeColumnDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableChangeColumnDesc.java
@@ -21,9 +21,9 @@ import java.util.Map;
import org.apache.hadoop.hive.ql.ddl.DDLTask2;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableWithConstraintsDesc;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
import org.apache.hadoop.hive.ql.ddl.table.constaint.Constraints;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.plan.DDLDesc.DDLDescWithWriteId;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
@@ -49,7 +49,7 @@ public class AlterTableChangeColumnDesc extends AbstractAlterTableWithConstraint
public AlterTableChangeColumnDesc(String tableName, Map<String, String> partitionSpec, boolean isCascade,
Constraints constraints, String oldColumnName, String newColumnName, String newColumnType,
String newColumnComment, boolean first, String afterColumn) throws SemanticException {
- super(AlterTableTypes.RENAME_COLUMN, tableName, partitionSpec, null, isCascade, false, null, constraints);
+ super(AlterTableType.RENAME_COLUMN, tableName, partitionSpec, null, isCascade, false, null, constraints);
this.oldColumnName = oldColumnName;
this.newColumnName = newColumnName;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableReplaceColumnsDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableReplaceColumnsDesc.java
index 3975f66..b4e7004 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableReplaceColumnsDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableReplaceColumnsDesc.java
@@ -23,9 +23,9 @@ import java.util.Map;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.ddl.DDLTask2;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.plan.DDLDesc.DDLDescWithWriteId;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
@@ -45,7 +45,7 @@ public class AlterTableReplaceColumnsDesc extends AbstractAlterTableDesc impleme
public AlterTableReplaceColumnsDesc(String tableName, Map<String, String> partitionSpec, boolean isCascade,
List<FieldSchema> newColumns) throws SemanticException {
- super(AlterTableTypes.REPLACE_COLUMNS, tableName, partitionSpec, null, isCascade, false, null);
+ super(AlterTableType.REPLACE_COLUMNS, tableName, partitionSpec, null, isCascade, false, null);
this.newColumns = newColumns;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableUpdateColumnsDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableUpdateColumnsDesc.java
index 18485c9..0acaa87 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableUpdateColumnsDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableUpdateColumnsDesc.java
@@ -22,8 +22,8 @@ import java.util.Map;
import org.apache.hadoop.hive.ql.ddl.DDLTask2;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.plan.DDLDesc.DDLDescWithWriteId;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
@@ -41,7 +41,7 @@ public class AlterTableUpdateColumnsDesc extends AbstractAlterTableDesc implemen
public AlterTableUpdateColumnsDesc(String tableName, Map<String, String> partitionSpec, boolean isCascade)
throws SemanticException {
- super(AlterTableTypes.UPDATE_COLUMNS, tableName, partitionSpec, null, isCascade, false, null);
+ super(AlterTableType.UPDATE_COLUMNS, tableName, partitionSpec, null, isCascade, false, null);
}
@Override
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/constaint/AlterTableAddConstraintDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/constaint/AlterTableAddConstraintDesc.java
index 2077c7d..35023db 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/constaint/AlterTableAddConstraintDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/constaint/AlterTableAddConstraintDesc.java
@@ -19,9 +19,9 @@ package org.apache.hadoop.hive.ql.ddl.table.constaint;
import org.apache.hadoop.hive.ql.ddl.DDLTask2;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableWithConstraintsDesc;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
@@ -38,7 +38,7 @@ public class AlterTableAddConstraintDesc extends AbstractAlterTableWithConstrain
public AlterTableAddConstraintDesc(String tableName, ReplicationSpec replicationSpec, Constraints constraints)
throws SemanticException {
- super(AlterTableTypes.ADD_CONSTRAINT, tableName, null, replicationSpec, false, false, null, constraints);
+ super(AlterTableType.ADD_CONSTRAINT, tableName, null, replicationSpec, false, false, null, constraints);
}
@Override
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetLocationDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableRenameDesc.java
similarity index 59%
copy from ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetLocationDesc.java
copy to ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableRenameDesc.java
index c918bb9..e98d528 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetLocationDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableRenameDesc.java
@@ -16,39 +16,38 @@
* limitations under the License.
*/
-package org.apache.hadoop.hive.ql.ddl.table.storage;
-
-import java.util.Map;
+package org.apache.hadoop.hive.ql.ddl.table.misc;
import org.apache.hadoop.hive.ql.ddl.DDLTask2;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
+import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
/**
- * DDL task description for ALTER TABLE ... SET LOCATION ... commands.
+ * DDL task description for ALTER TABLE ... RENAME TO ... commands.
*/
-@Explain(displayName = "Set Location", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
-public class AlterTableSetLocationDesc extends AbstractAlterTableDesc {
+@Explain(displayName = "Rename Table", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+public class AlterTableRenameDesc extends AbstractAlterTableDesc {
private static final long serialVersionUID = 1L;
static {
- DDLTask2.registerOperation(AlterTableSetLocationDesc.class, AlterTableSetLocationOperation.class);
+ DDLTask2.registerOperation(AlterTableRenameDesc.class, AlterTableRenameOperation.class);
}
- private final String location;
+ private final String newName;
- public AlterTableSetLocationDesc(String tableName, Map<String, String> partitionSpec, String location)
+ public AlterTableRenameDesc(String tableName, ReplicationSpec replicationSpec, boolean expectView, String newName)
throws SemanticException {
- super(AlterTableTypes.SET_LOCATION, tableName, partitionSpec, null, false, false, null);
- this.location = location;
+ super(AlterTableType.RENAME, tableName, null, replicationSpec, false, expectView, null);
+ this.newName = newName;
}
- @Explain(displayName = "location", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
- public String getLocation() {
- return location;
+ @Explain(displayName = "new table name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+ public String getNewName() {
+ return newName;
}
@Override
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableRenameOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableRenameOperation.java
new file mode 100644
index 0000000..4345c9e
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableRenameOperation.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.table.misc;
+
+import org.apache.hadoop.hive.ql.ddl.DDLOperationContext;
+import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableOperation;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.parse.repl.dump.Utils;
+
+/**
+ * Operation process of renaming a table.
+ */
+public class AlterTableRenameOperation extends AbstractAlterTableOperation {
+ private final AlterTableRenameDesc desc;
+
+ public AlterTableRenameOperation(DDLOperationContext context, AlterTableRenameDesc desc) {
+ super(context, desc);
+ this.desc = desc;
+ }
+
+ @Override
+ public int execute() throws HiveException {
+ String[] names = Utilities.getDbTableName(desc.getTableName());
+ if (Utils.isBootstrapDumpInProgress(context.getDb(), names[0])) {
+ LOG.error("DDLTask: Rename Table not allowed as bootstrap dump in progress");
+ throw new HiveException("Rename Table: Not allowed as bootstrap dump in progress");
+ }
+
+ return super.execute();
+ }
+
+ @Override
+ protected void doAlteration(Table table, Partition partition) throws HiveException {
+ table.setDbName(Utilities.getDatabaseName(desc.getNewName()));
+ table.setTableName(Utilities.getTableName(desc.getNewName()));
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSerdeDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetOwnerDesc.java
similarity index 58%
copy from ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSerdeDesc.java
copy to ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetOwnerDesc.java
index 861139d..29a0bcf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSerdeDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetOwnerDesc.java
@@ -16,39 +16,37 @@
* limitations under the License.
*/
-package org.apache.hadoop.hive.ql.ddl.table.storage;
-
-import java.util.Map;
+package org.apache.hadoop.hive.ql.ddl.table.misc;
import org.apache.hadoop.hive.ql.ddl.DDLTask2;
+import org.apache.hadoop.hive.ql.ddl.privilege.PrincipalDesc;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
/**
- * DDL task description for ALTER TABLE ... SET SERDE ... commands.
+ * DDL task description for ALTER TABLE ... SET OWNER ... commands.
*/
-@Explain(displayName = "Set Serde", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
-public class AlterTableSetSerdeDesc extends AbstractAlterTableDesc {
+@Explain(displayName = "Set Owner", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+public class AlterTableSetOwnerDesc extends AbstractAlterTableDesc {
private static final long serialVersionUID = 1L;
static {
- DDLTask2.registerOperation(AlterTableSetSerdeDesc.class, AlterTableSetSerdeOperation.class);
+ DDLTask2.registerOperation(AlterTableSetOwnerDesc.class, AlterTableSetOwnerOperation.class);
}
- private final String serdeName;
+ private final PrincipalDesc ownerPrincipal;
- public AlterTableSetSerdeDesc(String tableName, Map<String, String> partitionSpec, Map<String, String> props,
- String serdeName) throws SemanticException {
- super(AlterTableTypes.SET_SERDE, tableName, partitionSpec, null, false, false, props);
- this.serdeName = serdeName;
+ public AlterTableSetOwnerDesc(String tableName, PrincipalDesc ownerPrincipal) throws SemanticException {
+ super(AlterTableType.OWNER, tableName, null, null, false, false, null);
+ this.ownerPrincipal = ownerPrincipal;
}
- @Explain(displayName = "serde", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
- public String getSerdeName() {
- return serdeName;
+ @Explain(displayName = "owner principal", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+ public PrincipalDesc getOwnerPrincipal() {
+ return ownerPrincipal;
}
@Override
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetOwnerOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetOwnerOperation.java
new file mode 100644
index 0000000..eb7be24
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetOwnerOperation.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.table.misc;
+
+import org.apache.hadoop.hive.ql.ddl.DDLOperationContext;
+import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableOperation;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
+
+/**
+ * Operation process of setting the owner of a table.
+ */
+public class AlterTableSetOwnerOperation extends AbstractAlterTableOperation {
+ private final AlterTableSetOwnerDesc desc;
+
+ public AlterTableSetOwnerOperation(DDLOperationContext context, AlterTableSetOwnerDesc desc) {
+ super(context, desc);
+ this.desc = desc;
+ }
+
+ @Override
+ protected void doAlteration(Table table, Partition partition) throws HiveException {
+ if (desc.getOwnerPrincipal() != null) {
+ table.setOwner(desc.getOwnerPrincipal().getName());
+ table.setOwnerType(desc.getOwnerPrincipal().getType());
+ }
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetPropertiesDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetPropertiesDesc.java
new file mode 100644
index 0000000..6ef7ced
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetPropertiesDesc.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.table.misc;
+
+import java.util.Map;
+
+import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
+import org.apache.hadoop.hive.ql.ddl.DDLTask2;
+import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.Explain;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
+
+/**
+ * DDL task description for ALTER TABLE ... SET TBLPROPERTIES ... commands.
+ */
+@Explain(displayName = "Set Properties", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+public class AlterTableSetPropertiesDesc extends AbstractAlterTableDesc {
+ private static final long serialVersionUID = 1L;
+
+ static {
+ DDLTask2.registerOperation(AlterTableSetPropertiesDesc.class, AlterTableSetPropertiesOperation.class);
+ }
+
+ private final boolean isExplicitStatsUpdate;
+ private final boolean isFullAcidConversion;
+ private final EnvironmentContext environmentContext;
+
+ public AlterTableSetPropertiesDesc(String tableName, Map<String, String> partitionSpec,
+ ReplicationSpec replicationSpec, boolean expectView, Map<String, String> props, boolean isExplicitStatsUpdate,
+ boolean isFullAcidConversion, EnvironmentContext environmentContext) throws SemanticException {
+ super(AlterTableType.ADDPROPS, tableName, partitionSpec, replicationSpec, false, expectView, props);
+ this.isExplicitStatsUpdate = isExplicitStatsUpdate;
+ this.isFullAcidConversion = isFullAcidConversion;
+ this.environmentContext = environmentContext;
+ }
+
+ @Override
+ public EnvironmentContext getEnvironmentContext() {
+ return environmentContext;
+ }
+
+ @Override
+ public boolean mayNeedWriteId() {
+ return isExplicitStatsUpdate || AcidUtils.isToInsertOnlyTable(null, getProps()) ||
+ (AcidUtils.isTransactionalTable(getProps()) && !isFullAcidConversion);
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetPropertiesOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetPropertiesOperation.java
new file mode 100644
index 0000000..b0a763f
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetPropertiesOperation.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.table.misc;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang.BooleanUtils;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.ql.ddl.DDLOperationContext;
+import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableOperation;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.PartitionIterable;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.LoadMultiFilesDesc;
+import org.apache.hadoop.hive.ql.plan.MoveWork;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Operation process of setting properties of a table.
+ */
+public class AlterTableSetPropertiesOperation extends AbstractAlterTableOperation {
+ private final AlterTableSetPropertiesDesc desc;
+
+ public AlterTableSetPropertiesOperation(DDLOperationContext context, AlterTableSetPropertiesDesc desc) {
+ super(context, desc);
+ this.desc = desc;
+ }
+
+ @Override
+ protected void doAlteration(Table table, Partition partition) throws HiveException {
+ if (StatsSetupConst.USER.equals(environmentContext.getProperties().get(StatsSetupConst.STATS_GENERATED))) {
+ environmentContext.getProperties().remove(StatsSetupConst.DO_NOT_UPDATE_STATS);
+ }
+
+ if (partition != null) {
+ partition.getTPartition().getParameters().putAll(desc.getProps());
+ } else {
+ boolean isFromMmTable = AcidUtils.isInsertOnlyTable(table.getParameters());
+ Boolean isToMmTable = AcidUtils.isToInsertOnlyTable(table, desc.getProps());
+ if (!isFromMmTable && BooleanUtils.isTrue(isToMmTable)) {
+ if (!HiveConf.getBoolVar(context.getConf(), ConfVars.HIVE_MM_ALLOW_ORIGINALS)) {
+ List<Task<?>> mmTasks = generateAddMmTasks(table, desc.getWriteId());
+ for (Task<?> mmTask : mmTasks) {
+ context.getTask().addDependentTask(mmTask);
+ }
+ } else {
+ if (!table.getPartitionKeys().isEmpty()) {
+ PartitionIterable parts = new PartitionIterable(context.getDb(), table, null,
+ HiveConf.getIntVar(context.getConf(), ConfVars.METASTORE_BATCH_RETRIEVE_MAX));
+ for (Partition part : parts) {
+ checkMmLb(part);
+ }
+ } else {
+ checkMmLb(table);
+ }
+ }
+ } else if (isFromMmTable && BooleanUtils.isFalse(isToMmTable)) {
+ throw new HiveException("Cannot convert an ACID table to non-ACID");
+ }
+
+ // Converting to/from external table
+ String externalProp = desc.getProps().get("EXTERNAL");
+ if (externalProp != null) {
+ if (Boolean.parseBoolean(externalProp) && table.getTableType() == TableType.MANAGED_TABLE) {
+ table.setTableType(TableType.EXTERNAL_TABLE);
+ } else if (!Boolean.parseBoolean(externalProp) && table.getTableType() == TableType.EXTERNAL_TABLE) {
+ table.setTableType(TableType.MANAGED_TABLE);
+ }
+ }
+
+ table.getTTable().getParameters().putAll(desc.getProps());
+ }
+ }
+
+
+ private List<Task<?>> generateAddMmTasks(Table table, Long writeId) throws HiveException {
+ // We will move all the files in the table/partition directories into the first MM
+ // directory, then commit the first write ID.
+ if (writeId == null) {
+ throw new HiveException("Internal error - write ID not set for MM conversion");
+ }
+
+ List<Path> sources = new ArrayList<>();
+ List<Path> targets = new ArrayList<>();
+
+ int stmtId = 0;
+ String mmDir = AcidUtils.deltaSubdir(writeId, writeId, stmtId);
+
+ if (!table.getPartitionKeys().isEmpty()) {
+ PartitionIterable parts = new PartitionIterable(context.getDb(), table, null,
+ HiveConf.getIntVar(context.getConf(), ConfVars.METASTORE_BATCH_RETRIEVE_MAX));
+ for (Partition part : parts) {
+ checkMmLb(part);
+ Path source = part.getDataLocation();
+ Path target = new Path(source, mmDir);
+ sources.add(source);
+ targets.add(target);
+ Utilities.FILE_OP_LOGGER.trace("Will move " + source + " to " + target);
+ }
+ } else {
+ checkMmLb(table);
+ Path source = table.getDataLocation();
+ Path target = new Path(source, mmDir);
+ sources.add(source);
+ targets.add(target);
+ Utilities.FILE_OP_LOGGER.trace("Will move " + source + " to " + target);
+ }
+
+ // Don't set inputs and outputs - the locks have already been taken so it's pointless.
+ MoveWork mw = new MoveWork(null, null, null, null, false);
+ mw.setMultiFilesDesc(new LoadMultiFilesDesc(sources, targets, true, null, null));
+ return Lists.<Task<?>>newArrayList(TaskFactory.get(mw));
+ }
+
+ private void checkMmLb(Table table) throws HiveException {
+ if (!table.isStoredAsSubDirectories()) {
+ return;
+ }
+ // TODO [MM gap?]: by design; no-one seems to use LB tables. They will work, but not convert.
+ // It's possible to work around this by re-creating and re-inserting the table.
+ throw new HiveException("Converting list bucketed tables stored as subdirectories "
+ + " to MM is not supported. Please re-create a table in the desired format.");
+ }
+
+ private void checkMmLb(Partition partition) throws HiveException {
+ if (!partition.isStoredAsSubDirectories()) {
+ return;
+ }
+ throw new HiveException("Converting list bucketed tables stored as subdirectories "
+ + " to MM is not supported. Please re-create a table in the desired format.");
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSortedDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableTouchDesc.java
similarity index 50%
copy from ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSortedDesc.java
copy to ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableTouchDesc.java
index 11e8bf3..92f498d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSortedDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableTouchDesc.java
@@ -16,34 +16,39 @@
* limitations under the License.
*/
-package org.apache.hadoop.hive.ql.ddl.table.storage;
+package org.apache.hadoop.hive.ql.ddl.table.misc;
import java.util.Map;
+import org.apache.hadoop.hive.ql.ddl.DDLDesc;
import org.apache.hadoop.hive.ql.ddl.DDLTask2;
-import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc;
-import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
/**
- * DDL task description for ALTER TABLE ... NOT SORTED commands.
+ * DDL task description for ALTER TABLE ... TOUCH [PARTITION ...] commands.
*/
-@Explain(displayName = "Not Sorted", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
-public class AlterTableNotSortedDesc extends AbstractAlterTableDesc {
- private static final long serialVersionUID = 1L;
-
+@Explain(displayName = "Touch", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+public class AlterTableTouchDesc implements DDLDesc {
static {
- DDLTask2.registerOperation(AlterTableNotSortedDesc.class, AlterTableNotSortedOperation.class);
+ DDLTask2.registerOperation(AlterTableTouchDesc.class, AlterTableTouchOperation.class);
+ }
+
+ private final String tableName;
+ private Map<String, String> partitionSpec;
+
+ public AlterTableTouchDesc(String tableName, Map<String, String> partitionSpec) {
+ this.tableName = tableName;
+ this.partitionSpec = partitionSpec;
}
- public AlterTableNotSortedDesc(String tableName, Map<String, String> partitionSpec) throws SemanticException {
- super(AlterTableTypes.NOT_SORTED, tableName, partitionSpec, null, false, false, null);
+ @Explain(displayName = "table name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+ public String getTableName() {
+ return tableName;
}
- @Override
- public boolean mayNeedWriteId() {
- return false;
+ @Explain(displayName = "partition spec", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+ public Map<String, String> getPartitionSpec() {
+ return partitionSpec;
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableTouchOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableTouchOperation.java
new file mode 100644
index 0000000..4180f62
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableTouchOperation.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.table.misc;
+
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.ql.ddl.DDLOperation;
+import org.apache.hadoop.hive.ql.ddl.DDLOperationContext;
+import org.apache.hadoop.hive.ql.ddl.DDLUtils;
+import org.apache.hadoop.hive.ql.hooks.ReadEntity;
+import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
+
+/**
+ * Operation process of touching a table.
+ */
+public class AlterTableTouchOperation extends DDLOperation {
+ private final AlterTableTouchDesc desc;
+
+ public AlterTableTouchOperation(DDLOperationContext context, AlterTableTouchDesc desc) {
+ super(context);
+ this.desc = desc;
+ }
+
+ @Override
+ public int execute() throws HiveException {
+ // TODO: catalog
+ Table table = context.getDb().getTable(desc.getTableName());
+ EnvironmentContext environmentContext = new EnvironmentContext();
+ environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE);
+
+ if (desc.getPartitionSpec() == null) {
+ context.getDb().alterTable(table, false, environmentContext, true);
+ context.getWork().getInputs().add(new ReadEntity(table));
+ DDLUtils.addIfAbsentByName(new WriteEntity(table, WriteEntity.WriteType.DDL_NO_LOCK), context);
+ } else {
+ Partition part = context.getDb().getPartition(table, desc.getPartitionSpec(), false);
+ if (part == null) {
+ throw new HiveException("Specified partition does not exist");
+ }
+ try {
+ context.getDb().alterPartition(table.getCatalogName(), table.getDbName(), table.getTableName(), part,
+ environmentContext, true);
+ } catch (InvalidOperationException e) {
+ throw new HiveException(e);
+ }
+ context.getWork().getInputs().add(new ReadEntity(part));
+ DDLUtils.addIfAbsentByName(new WriteEntity(part, WriteEntity.WriteType.DDL_NO_LOCK), context);
+ }
+ return 0;
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableUnsetPropertiesDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableUnsetPropertiesDesc.java
new file mode 100644
index 0000000..a156972
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableUnsetPropertiesDesc.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.table.misc;
+
+import java.util.Map;
+
+import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
+import org.apache.hadoop.hive.ql.ddl.DDLTask2;
+import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
+import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.Explain;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
+
+/**
+ * DDL task description for ALTER TABLE ... UNSET TBLPROPERTIES [IF EXISTS] ... commands.
+ */
+@Explain(displayName = "Unset Properties", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+public class AlterTableUnsetPropertiesDesc extends AbstractAlterTableDesc {
+ private static final long serialVersionUID = 1L;
+
+ static {
+ DDLTask2.registerOperation(AlterTableUnsetPropertiesDesc.class, AlterTableUnsetPropertiesOperation.class);
+ }
+
+ private final boolean isExplicitStatsUpdate;
+ private final EnvironmentContext environmentContext;
+
+ public AlterTableUnsetPropertiesDesc(String tableName, Map<String, String> partitionSpec,
+ ReplicationSpec replicationSpec, boolean expectView, Map<String, String> props, boolean isExplicitStatsUpdate,
+ EnvironmentContext environmentContext) throws SemanticException {
+ super(AlterTableType.DROPPROPS, tableName, partitionSpec, replicationSpec, false, expectView, props);
+ this.isExplicitStatsUpdate = isExplicitStatsUpdate;
+ this.environmentContext = environmentContext;
+ }
+
+ @Override
+ public EnvironmentContext getEnvironmentContext() {
+ return environmentContext;
+ }
+
+ @Override
+ public boolean mayNeedWriteId() {
+ return isExplicitStatsUpdate;
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableUnsetPropertiesOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableUnsetPropertiesOperation.java
new file mode 100644
index 0000000..0e69a9a
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableUnsetPropertiesOperation.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.table.misc;
+
+import java.util.Set;
+
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.ql.ddl.DDLOperationContext;
+import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableOperation;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
+
+/**
+ * Operation process of unsetting properties of a table.
+ */
+public class AlterTableUnsetPropertiesOperation extends AbstractAlterTableOperation {
+ private final AlterTableUnsetPropertiesDesc desc;
+
+ public AlterTableUnsetPropertiesOperation(DDLOperationContext context, AlterTableUnsetPropertiesDesc desc) {
+ super(context, desc);
+ this.desc = desc;
+ }
+
+ @Override
+ protected void doAlteration(Table table, Partition partition) throws HiveException {
+ if (StatsSetupConst.USER.equals(environmentContext.getProperties().get(StatsSetupConst.STATS_GENERATED))) {
+ // drop a stats parameter, which triggers recompute stats update automatically
+ environmentContext.getProperties().remove(StatsSetupConst.DO_NOT_UPDATE_STATS);
+ }
+
+ if (partition == null) {
+ Set<String> removedSet = desc.getProps().keySet();
+ boolean isFromMmTable = AcidUtils.isInsertOnlyTable(table.getParameters());
+ boolean isRemoved = AcidUtils.isRemovedInsertOnlyTable(removedSet);
+ if (isFromMmTable && isRemoved) {
+ throw new HiveException("Cannot convert an ACID table to non-ACID");
+ }
+
+ // Check if external table property being removed
+ if (removedSet.contains("EXTERNAL") && table.getTableType() == TableType.EXTERNAL_TABLE) {
+ table.setTableType(TableType.MANAGED_TABLE);
+ }
+ }
+ for (String key : desc.getProps().keySet()) {
+ if (partition != null) {
+ partition.getTPartition().getParameters().remove(key);
+ } else {
+ table.getTTable().getParameters().remove(key);
+ }
+ }
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSortedDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableArchiveDesc.java
similarity index 52%
copy from ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSortedDesc.java
copy to ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableArchiveDesc.java
index 11e8bf3..67aabd5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSortedDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableArchiveDesc.java
@@ -20,30 +20,35 @@ package org.apache.hadoop.hive.ql.ddl.table.storage;
import java.util.Map;
+import org.apache.hadoop.hive.ql.ddl.DDLDesc;
import org.apache.hadoop.hive.ql.ddl.DDLTask2;
-import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc;
-import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
/**
- * DDL task description for ALTER TABLE ... NOT SORTED commands.
+ * DDL task description for ALTER TABLE ... ARCHIVE [PARTITION ...] commands.
*/
-@Explain(displayName = "Not Sorted", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
-public class AlterTableNotSortedDesc extends AbstractAlterTableDesc {
- private static final long serialVersionUID = 1L;
-
+@Explain(displayName = "Archive", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+public class AlterTableArchiveDesc implements DDLDesc {
static {
- DDLTask2.registerOperation(AlterTableNotSortedDesc.class, AlterTableNotSortedOperation.class);
+ DDLTask2.registerOperation(AlterTableArchiveDesc.class, AlterTableArchiveOperation.class);
+ }
+
+ private final String tableName;
+ private final Map<String, String> partitionSpec;
+
+ public AlterTableArchiveDesc(String tableName, Map<String, String> partitionSpec) {
+ this.tableName = tableName;
+ this.partitionSpec = partitionSpec;
}
- public AlterTableNotSortedDesc(String tableName, Map<String, String> partitionSpec) throws SemanticException {
- super(AlterTableTypes.NOT_SORTED, tableName, partitionSpec, null, false, false, null);
+ @Explain(displayName = "table name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+ public String getTableName() {
+ return tableName;
}
- @Override
- public boolean mayNeedWriteId() {
- return false;
+ @Explain(displayName = "partition spec", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+ public Map<String, String> getPartitionSpec() {
+ return partitionSpec;
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableArchiveOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableArchiveOperation.java
new file mode 100644
index 0000000..d1f2a69
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableArchiveOperation.java
@@ -0,0 +1,318 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.table.storage;
+
+import org.apache.hadoop.hive.ql.ddl.DDLOperationContext;
+import org.apache.hadoop.hive.ql.exec.ArchiveUtils;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.io.HdfsUtils;
+import org.apache.hadoop.hive.ql.exec.ArchiveUtils.PartSpecInfo;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.ql.ddl.DDLOperation;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.tools.HadoopArchives;
+import org.apache.hadoop.util.ToolRunner;
+
+import com.google.common.collect.ImmutableList;
+
+import static org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableArchiveUtils.ARCHIVE_NAME;
+
+/**
+ * Operation process of archiving a table.
+ */
+public class AlterTableArchiveOperation extends DDLOperation {
+ private final AlterTableArchiveDesc desc;
+
+ public AlterTableArchiveOperation(DDLOperationContext context, AlterTableArchiveDesc desc) {
+ super(context);
+ this.desc = desc;
+ }
+
+ @Override
+ public int execute() throws HiveException {
+ Table table = context.getDb().getTable(desc.getTableName());
+ if (table.getTableType() != TableType.MANAGED_TABLE) {
+ throw new HiveException("ARCHIVE can only be performed on managed tables");
+ }
+
+ Map<String, String> partitionSpec = desc.getPartitionSpec();
+ PartSpecInfo partitionSpecInfo = PartSpecInfo.create(table, partitionSpec);
+ List<Partition> partitions = context.getDb().getPartitions(table, partitionSpec);
+
+ Path originalDir = getOriginalDir(table, partitionSpecInfo, partitions);
+ Path intermediateArchivedDir = AlterTableArchiveUtils.getInterMediateDir(originalDir, context.getConf(),
+ ConfVars.METASTORE_INT_ARCHIVED);
+ Path intermediateOriginalDir = AlterTableArchiveUtils.getInterMediateDir(originalDir, context.getConf(),
+ ConfVars.METASTORE_INT_ORIGINAL);
+
+ context.getConsole().printInfo("intermediate.archived is " + intermediateArchivedDir.toString());
+ context.getConsole().printInfo("intermediate.original is " + intermediateOriginalDir.toString());
+
+ checkIfAlreadyArchived(partitionSpecInfo, partitions);
+ boolean recovery = isRecovery(intermediateArchivedDir, intermediateOriginalDir);
+
+ FileSystem fs = null;
+ try {
+ fs = originalDir.getFileSystem(context.getConf());
+ } catch (IOException e) {
+ throw new HiveException(e);
+ }
+
+ // The following steps seem roundabout, but they are meant to aid in recovery if a failure occurs and to keep a
+ // consistent state in the FS
+
+ // If the intermediate directory exists, we assume the dir is good to use as it's creation is atomic (move)
+ if (!recovery) {
+ Path tmpPath = createArchiveInTmpDir(table, partitionSpecInfo, originalDir);
+ moveTmpDirToIntermediateDir(intermediateArchivedDir, fs, tmpPath);
+ } else {
+ if (HdfsUtils.pathExists(intermediateArchivedDir, context.getConf())) {
+ context.getConsole().printInfo("Intermediate archive directory " + intermediateArchivedDir +
+ " already exists. Assuming it contains an archived version of the partition");
+ }
+ }
+
+ moveOriginalDirToIntermediateDir(originalDir, intermediateOriginalDir, fs);
+
+ // If there's a failure from here to when the metadata is updated, there will be no data in the partition, or an
+ // error while trying to read the partition (if the archive files have been moved to the original partition
+ // directory.) But re-running the archive command will allow recovery
+
+ moveIntermediateArchivedDirToOriginalParent(originalDir, intermediateArchivedDir, fs);
+
+ writeArchivationToMetastore(partitionSpecInfo, partitions, originalDir);
+
+ // If a failure occurs here, the directory containing the original files will not be deleted. The user will run
+ // ARCHIVE again to clear this up. The presence of these files are used to indicate whether the original partition
+ // directory contains archived or unarchived files.
+
+ deleteIntermediateOriginalDir(table, intermediateOriginalDir);
+
+ if (recovery) {
+ context.getConsole().printInfo("Recovery after ARCHIVE succeeded");
+ }
+
+ return 0;
+ }
+
+ private Path getOriginalDir(Table table, PartSpecInfo partitionSpecInfo, List<Partition> partitions)
+ throws HiveException {
+ // when we have partial partitions specification we must assume partitions lie in standard place -
+ // if they were in custom locations putting them into one archive would involve mass amount of copying
+ // in full partition specification case we allow custom locations to keep backward compatibility
+ if (partitions.isEmpty()) {
+ throw new HiveException("No partition matches the specification");
+ } else if (partitionSpecInfo.values.size() != table.getPartCols().size()) {
+ // for partial specifications we need partitions to follow the scheme
+ for (Partition partition : partitions){
+ if (AlterTableArchiveUtils.partitionInCustomLocation(table, partition)) {
+ throw new HiveException(String.format("ARCHIVE cannot run for partition groups with custom locations like %s",
+ partition.getLocation()));
+ }
+ }
+ return partitionSpecInfo.createPath(table);
+ } else {
+ Partition p = partitions.get(0);
+ // partition can be archived if during recovery
+ if (ArchiveUtils.isArchived(p)) {
+ return new Path(AlterTableArchiveUtils.getOriginalLocation(p));
+ } else {
+ return p.getDataLocation();
+ }
+ }
+ }
+
+ private void checkIfAlreadyArchived(PartSpecInfo partitionSpecInfo, List<Partition> partitions) throws HiveException {
+ // we checked if partitions matching specification are marked as archived in the metadata; if they are then
+ // throw an exception
+ for (Partition partition : partitions) {
+ if (ArchiveUtils.isArchived(partition)) {
+ if (ArchiveUtils.getArchivingLevel(partition) != partitionSpecInfo.values.size()) {
+ String name = ArchiveUtils.getPartialName(partition, ArchiveUtils.getArchivingLevel(partition));
+ throw new HiveException(String.format("Conflict with existing archive %s", name));
+ } else {
+ throw new HiveException("Partition(s) already archived");
+ }
+ }
+ }
+ }
+
+ private boolean isRecovery(Path intermediateArchivedDir, Path intermediateOriginalDir) throws HiveException {
+ if (HdfsUtils.pathExists(intermediateArchivedDir, context.getConf()) ||
+ HdfsUtils.pathExists(intermediateOriginalDir, context.getConf())) {
+ context.getConsole().printInfo("Starting recovery after failed ARCHIVE");
+ return true;
+ }
+ return false;
+ }
+
+ private Path createArchiveInTmpDir(Table table, PartSpecInfo partitionSpecInfo, Path originalDir)
+ throws HiveException {
+ // First create the archive in a tmp dir so that if the job fails, the bad files don't pollute the filesystem
+ Path tmpPath = new Path(context.getDriverContext().getCtx().getExternalTmpPath(originalDir), "partlevel");
+
+ // Create the Hadoop archive
+ context.getConsole().printInfo("Creating " + ARCHIVE_NAME + " for " + originalDir.toString() + " in " + tmpPath);
+ context.getConsole().printInfo("Please wait... (this may take a while)");
+ try {
+ int maxJobNameLength = context.getConf().getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH);
+ String jobName = String.format("Archiving %s@%s", table.getTableName(), partitionSpecInfo.getName());
+ jobName = Utilities.abbreviate(jobName, maxJobNameLength - 6);
+ context.getConf().set(MRJobConfig.JOB_NAME, jobName);
+
+ HadoopArchives har = new HadoopArchives(context.getConf());
+ List<String> args = ImmutableList.of("-archiveName", ARCHIVE_NAME, "-p", originalDir.toString(),
+ tmpPath.toString());
+
+ int ret = ToolRunner.run(har, args.toArray(new String[0]));
+ if (ret != 0) {
+ throw new HiveException("Error while creating HAR");
+ }
+ } catch (Exception e) {
+ throw new HiveException(e);
+ }
+
+ return tmpPath;
+ }
+
+ /**
+ * Move from the tmp dir to an intermediate directory, in the same level as the partition directory.
+ * e.g. .../hr=12-intermediate-archived
+ */
+ private void moveTmpDirToIntermediateDir(Path intermediateArchivedDir, FileSystem fs, Path tmpPath)
+ throws HiveException {
+ try {
+ context.getConsole().printInfo("Moving " + tmpPath + " to " + intermediateArchivedDir);
+ if (HdfsUtils.pathExists(intermediateArchivedDir, context.getConf())) {
+ throw new HiveException("The intermediate archive directory already exists.");
+ }
+ fs.rename(tmpPath, intermediateArchivedDir);
+ } catch (IOException e) {
+ throw new HiveException("Error while moving tmp directory");
+ }
+ }
+
+ /**
+ * Move the original parent directory to the intermediate original directory if the move hasn't been made already.
+ */
+ private void moveOriginalDirToIntermediateDir(Path originalDir, Path intermediateOriginalDir, FileSystem fs)
+ throws HiveException {
+ // If we get to here, we know that we've archived the partition files, but they may be in the original partition
+ // location, or in the intermediate original dir.
+ if (!HdfsUtils.pathExists(intermediateOriginalDir, context.getConf())) {
+ context.getConsole().printInfo("Moving " + originalDir + " to " + intermediateOriginalDir);
+ moveDir(fs, originalDir, intermediateOriginalDir);
+ } else {
+ context.getConsole().printInfo(intermediateOriginalDir + " already exists. " +
+ "Assuming it contains the original files in the partition");
+ }
+ }
+
+ /**
+ * Move the intermediate archived directory to the original parent directory.
+ */
+ private void moveIntermediateArchivedDirToOriginalParent(Path originalDir, Path intermediateArchivedDir,
+ FileSystem fs) throws HiveException {
+ if (!HdfsUtils.pathExists(originalDir, context.getConf())) {
+ context.getConsole().printInfo("Moving " + intermediateArchivedDir + " to " + originalDir);
+ moveDir(fs, intermediateArchivedDir, originalDir);
+ } else {
+ context.getConsole().printInfo(originalDir + " already exists. Assuming it contains the archived version of "
+ + "the partition");
+ }
+ }
+
+ private void moveDir(FileSystem fs, Path from, Path to) throws HiveException {
+ try {
+ if (!fs.rename(from, to)) {
+ throw new HiveException("Moving " + from + " to " + to + " failed!");
+ }
+ } catch (IOException e) {
+ throw new HiveException(e);
+ }
+ }
+
+ /**
+ * Record the previous changes in the metastore.
+ */
+ private void writeArchivationToMetastore(PartSpecInfo partitionSpecInfo, List<Partition> partitions, Path originalDir)
+ throws HiveException {
+ try {
+ URI archiveUri = new Path(originalDir, ARCHIVE_NAME).toUri();
+ URI originalUri = ArchiveUtils.addSlash(originalDir.toUri());
+ ArchiveUtils.HarPathHelper harHelper = new ArchiveUtils.HarPathHelper(context.getConf(), archiveUri, originalUri);
+
+ for (Partition partition : partitions) {
+ URI originalPartitionUri = ArchiveUtils.addSlash(partition.getDataLocation().toUri());
+ URI harPartitionDir = harHelper.getHarUri(originalPartitionUri);
+ StringBuilder authority = new StringBuilder();
+ if (harPartitionDir.getUserInfo() != null) {
+ authority.append(harPartitionDir.getUserInfo()).append("@");
+ }
+ authority.append(harPartitionDir.getHost());
+ if (harPartitionDir.getPort() != -1) {
+ authority.append(":").append(harPartitionDir.getPort());
+ }
+
+ // make in Path to ensure no slash at the end
+ Path harPath = new Path(harPartitionDir.getScheme(), authority.toString(), harPartitionDir.getPath());
+ setArchived(partition, harPath, partitionSpecInfo.values.size());
+ // TODO: catalog
+ context.getDb().alterPartition(desc.getTableName(), partition, null, true);
+ }
+ } catch (Exception e) {
+ throw new HiveException("Unable to change the partition info for HAR", e);
+ }
+ }
+
+ /**
+ * Sets the appropriate attributes in the supplied Partition object to mark
+ * it as archived. Note that the metastore is not touched - a separate
+ * call to alter_partition is needed.
+ *
+ * @param p - the partition object to modify
+ * @param harPath - new location of partition (har schema URI)
+ */
+ private void setArchived(Partition p, Path harPath, int level) {
+ assert(!ArchiveUtils.isArchived(p));
+ AlterTableArchiveUtils.setIsArchived(p, true, level);
+ AlterTableArchiveUtils.setOriginalLocation(p, p.getLocation());
+ p.setLocation(harPath.toString());
+ }
+
+ private void deleteIntermediateOriginalDir(Table table, Path intermediateOriginalDir) throws HiveException {
+ if (HdfsUtils.pathExists(intermediateOriginalDir, context.getConf())) {
+ AlterTableArchiveUtils.deleteDir(intermediateOriginalDir, context.getDb().getDatabase(table.getDbName()),
+ context.getConf());
+ }
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableArchiveUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableArchiveUtils.java
new file mode 100644
index 0000000..ebac6f6
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableArchiveUtils.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.table.storage;
+
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.hive.ql.exec.ArchiveUtils;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
+
+/**
+ * Utilities for archiving.
+ */
+final class AlterTableArchiveUtils {
+ private AlterTableArchiveUtils() {
+ throw new UnsupportedOperationException("ArchiveUtils should not be instantiated");
+ }
+
+ static final String ARCHIVE_NAME = "data.har";
+
+ /**
+ * Returns original partition of archived partition, null for unarchived one.
+ */
+ static String getOriginalLocation(Partition partition) {
+ Map<String, String> params = partition.getParameters();
+ return params.get(hive_metastoreConstants.ORIGINAL_LOCATION);
+ }
+
+ /**
+ * Sets original location of partition which is to be archived.
+ */
+ static void setOriginalLocation(Partition partition, String loc) {
+ Map<String, String> params = partition.getParameters();
+ if (loc == null) {
+ params.remove(hive_metastoreConstants.ORIGINAL_LOCATION);
+ } else {
+ params.put(hive_metastoreConstants.ORIGINAL_LOCATION, loc);
+ }
+ }
+
+ /**
+ * Checks in partition is in custom (not-standard) location.
+ * @param table - table in which partition is
+ * @param partition - partition
+ * @return true if partition location is custom, false if it is standard
+ */
+ static boolean partitionInCustomLocation(Table table, Partition partition) throws HiveException {
+ String subdir = null;
+ try {
+ subdir = Warehouse.makePartName(table.getPartCols(), partition.getValues());
+ } catch (MetaException e) {
+ throw new HiveException("Unable to get partition's directory", e);
+ }
+
+ Path tableDir = table.getDataLocation();
+ if (tableDir == null) {
+ throw new HiveException("Table has no location set");
+ }
+
+ String standardLocation = new Path(tableDir, subdir).toString();
+ if (ArchiveUtils.isArchived(partition)) {
+ return !getOriginalLocation(partition).equals(standardLocation);
+ } else {
+ return !partition.getLocation().equals(standardLocation);
+ }
+ }
+
+ static Path getInterMediateDir(Path dir, Configuration conf, ConfVars suffixConfig) {
+ String intermediateDirSuffix = HiveConf.getVar(conf, suffixConfig);
+ return new Path(dir.getParent(), dir.getName() + intermediateDirSuffix);
+ }
+
+ static void deleteDir(Path dir, Database db, Configuration conf) throws HiveException {
+ try {
+ Warehouse wh = new Warehouse(conf);
+ wh.deleteDir(dir, true, db);
+ } catch (MetaException e) {
+ throw new HiveException(e);
+ }
+ }
+
+ /**
+ * Sets archiving flag locally; it has to be pushed into metastore.
+ * @param partition partition to set flag
+ * @param state desired state of IS_ARCHIVED flag
+ * @param level desired level for state == true, anything for false
+ */
+ static void setIsArchived(Partition partition, boolean state, int level) {
+ Map<String, String> params = partition.getParameters();
+ if (state) {
+ params.put(hive_metastoreConstants.IS_ARCHIVED, "true");
+ params.put(ArchiveUtils.ARCHIVING_LEVEL, Integer.toString(level));
+ } else {
+ params.remove(hive_metastoreConstants.IS_ARCHIVED);
+ params.remove(ArchiveUtils.ARCHIVING_LEVEL);
+ }
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableClusteredByDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableClusteredByDesc.java
index 8aab47b..43862b0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableClusteredByDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableClusteredByDesc.java
@@ -25,8 +25,8 @@ import java.util.stream.Collectors;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.ql.ddl.DDLTask2;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
import org.apache.hadoop.hive.ql.util.DirectionUtils;
@@ -48,7 +48,7 @@ public class AlterTableClusteredByDesc extends AbstractAlterTableDesc {
public AlterTableClusteredByDesc(String tableName, Map<String, String> partitionSpec, int numberBuckets,
List<String> bucketColumns, List<Order> sortColumns) throws SemanticException {
- super(AlterTableTypes.CLUSTERED_BY, tableName, partitionSpec, null, false, false, null);
+ super(AlterTableType.CLUSTERED_BY, tableName, partitionSpec, null, false, false, null);
this.numberBuckets = numberBuckets;
this.bucketColumns = bucketColumns;
this.sortColumns = sortColumns;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableCompactDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableCompactDesc.java
new file mode 100644
index 0000000..3ac9051
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableCompactDesc.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.table.storage;
+
+import java.util.Map;
+
+import org.apache.hadoop.hive.ql.ddl.DDLDesc;
+import org.apache.hadoop.hive.ql.ddl.DDLTask2;
+import org.apache.hadoop.hive.ql.plan.Explain;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
+
+/**
+ * DDL task description for ALTER TABLE ... [PARTITION ... ] COMPACT 'major|minor' [WITH OVERWRITE TBLPROPERTIES ...]
+ * commands.
+ */
+@Explain(displayName = "Compact", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+public class AlterTableCompactDesc implements DDLDesc {
+ static {
+ DDLTask2.registerOperation(AlterTableCompactDesc.class, AlterTableCompactOperation.class);
+ }
+
+ private final String tableName;
+ private final Map<String, String> partitionSpec;
+ private final String compactionType;
+ private final boolean isBlocking;
+ private final Map<String, String> properties;
+
+ public AlterTableCompactDesc(String tableName, Map<String, String> partitionSpec, String compactionType,
+ boolean isBlocking, Map<String, String> properties) {
+ this.tableName = tableName;
+ this.partitionSpec = partitionSpec;
+ this.compactionType = compactionType;
+ this.isBlocking = isBlocking;
+ this.properties = properties;
+ }
+
+ @Explain(displayName = "table name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+ public String getTableName() {
+ return tableName;
+ }
+
+ @Explain(displayName = "partition spec", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+ public Map<String, String> getPartitionSpec() {
+ return partitionSpec;
+ }
+
+ @Explain(displayName = "compaction type", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+ public String getCompactionType() {
+ return compactionType;
+ }
+
+ @Explain(displayName = "blocking", displayOnlyOnTrue = true,
+ explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+ public boolean isBlocking() {
+ return isBlocking;
+ }
+
+ @Explain(displayName = "properties", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+ public Map<String, String> getProperties() {
+ return properties;
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableCompactOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableCompactOperation.java
new file mode 100644
index 0000000..4e3e2c5
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableCompactOperation.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.table.storage;
+
+import org.apache.hadoop.hive.ql.ddl.DDLOperationContext;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hive.metastore.api.CompactionResponse;
+import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
+import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement;
+import org.apache.hadoop.hive.metastore.txn.TxnStore;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.ddl.DDLOperation;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
+
+/**
+ * Operation process of compacting a table.
+ */
+public class AlterTableCompactOperation extends DDLOperation {
+ private static final int FIVE_MINUTES_IN_MILLIES = 5*60*1000;
+ private final AlterTableCompactDesc desc;
+
+ public AlterTableCompactOperation(DDLOperationContext context, AlterTableCompactDesc desc) {
+ super(context);
+ this.desc = desc;
+ }
+
+ @Override
+ public int execute() throws HiveException {
+ Table table = context.getDb().getTable(desc.getTableName());
+ if (!AcidUtils.isTransactionalTable(table)) {
+ throw new HiveException(ErrorMsg.NONACID_COMPACTION_NOT_SUPPORTED, table.getDbName(), table.getTableName());
+ }
+
+ String partitionName = getPartitionName(table);
+
+ CompactionResponse resp = compact(table, partitionName);
+
+ if (desc.isBlocking() && resp.isAccepted()) {
+ waitForCompactionToFinish(resp);
+ }
+
+ return 0;
+ }
+
+ private String getPartitionName(Table table) throws HiveException {
+ String partitionName = null;
+ if (desc.getPartitionSpec() == null) {
+ if (table.isPartitioned()) { // Compaction can only be done on the whole table if the table is non-partitioned.
+ throw new HiveException(ErrorMsg.NO_COMPACTION_PARTITION);
+ }
+ } else {
+ Map<String, String> partitionSpec = desc.getPartitionSpec();
+ List<Partition> partitions = context.getDb().getPartitions(table, partitionSpec);
+ if (partitions.size() > 1) {
+ throw new HiveException(ErrorMsg.TOO_MANY_COMPACTION_PARTITIONS);
+ } else if (partitions.size() == 0) {
+ throw new HiveException(ErrorMsg.INVALID_PARTITION_SPEC);
+ }
+ partitionName = partitions.get(0).getName();
+ }
+ return partitionName;
+ }
+
+ private CompactionResponse compact(Table table, String partitionName) throws HiveException {
+ CompactionResponse resp = context.getDb().compact2(table.getDbName(), table.getTableName(), partitionName,
+ desc.getCompactionType(), desc.getProperties());
+ if (resp.isAccepted()) {
+ context.getConsole().printInfo("Compaction enqueued with id " + resp.getId());
+ } else {
+ context.getConsole().printInfo("Compaction already enqueued with id " + resp.getId() + "; State is " +
+ resp.getState());
+ }
+ return resp;
+ }
+
+ private void waitForCompactionToFinish(CompactionResponse resp) throws HiveException {
+ StringBuilder progressDots = new StringBuilder();
+ long waitTimeMs = 1000;
+ wait: while (true) {
+ //double wait time until 5min
+ waitTimeMs = waitTimeMs*2;
+ waitTimeMs = Math.max(waitTimeMs, FIVE_MINUTES_IN_MILLIES);
+ try {
+ Thread.sleep(waitTimeMs);
+ } catch (InterruptedException ex) {
+ context.getConsole().printInfo("Interrupted while waiting for compaction with id=" + resp.getId());
+ break;
+ }
+
+ //this could be expensive when there are a lot of compactions....
+ //todo: update to search by ID once HIVE-13353 is done
+ ShowCompactResponse allCompactions = context.getDb().showCompactions();
+ for (ShowCompactResponseElement compaction : allCompactions.getCompacts()) {
+ if (resp.getId() != compaction.getId()) {
+ continue;
+ }
+
+ switch (compaction.getState()) {
+ case TxnStore.WORKING_RESPONSE:
+ case TxnStore.INITIATED_RESPONSE:
+ //still working
+ context.getConsole().printInfo(progressDots.toString());
+ progressDots.append(".");
+ continue wait;
+ default:
+ //done
+ context.getConsole().printInfo("Compaction with id " + resp.getId() + " finished with status: " +
+ compaction.getState());
+ break wait;
+ }
+ }
+ }
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateDesc.java
similarity index 77%
rename from ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java
rename to ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateDesc.java
index 215da93..fd0ccb4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateDesc.java
@@ -16,25 +16,32 @@
* limitations under the License.
*/
-package org.apache.hadoop.hive.ql.parse;
+package org.apache.hadoop.hive.ql.ddl.table.storage;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.ddl.DDLDesc;
+import org.apache.hadoop.hive.ql.ddl.DDLTask2;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.ListBucketingCtx;
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
-
-@Explain(displayName = "Alter Table Partition Merge Files", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
-public class AlterTablePartMergeFilesDesc {
+/**
+ * DDL task description for ALTER TABLE ... [PARTITION ... ] CONCATENATE commands.
+ */
+@Explain(displayName = "Concatenate", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+public class AlterTableConcatenateDesc implements DDLDesc {
+ static {
+ DDLTask2.registerOperation(AlterTableConcatenateDesc.class, AlterTableConcatenateOperation.class);
+ }
private String tableName;
- private HashMap<String, String> partSpec;
+ private Map<String, String> partSpec;
private ListBucketingCtx lbCtx; // context for list bucketing.
private List<Path> inputDir = new ArrayList<Path>();
@@ -42,8 +49,8 @@ public class AlterTablePartMergeFilesDesc {
private Class<? extends InputFormat> inputFormatClass;
private TableDesc tableDesc;
- public AlterTablePartMergeFilesDesc(String tableName,
- HashMap<String, String> partSpec) {
+ public AlterTableConcatenateDesc(String tableName,
+ Map<String, String> partSpec) {
this.tableName = tableName;
this.partSpec = partSpec;
}
@@ -53,19 +60,11 @@ public class AlterTablePartMergeFilesDesc {
return tableName;
}
- public void setTableName(String tableName) {
- this.tableName = tableName;
- }
-
@Explain(displayName = "partition desc")
- public HashMap<String, String> getPartSpec() {
+ public Map<String, String> getPartSpec() {
return partSpec;
}
- public void setPartSpec(HashMap<String, String> partSpec) {
- this.partSpec = partSpec;
- }
-
public Path getOutputDir() {
return outputDir;
}
@@ -82,16 +81,10 @@ public class AlterTablePartMergeFilesDesc {
this.inputDir = inputDir;
}
- /**
- * @return the lbCtx
- */
public ListBucketingCtx getLbCtx() {
return lbCtx;
}
- /**
- * @param lbCtx the lbCtx to set
- */
public void setLbCtx(ListBucketingCtx lbCtx) {
this.lbCtx = lbCtx;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateOperation.java
new file mode 100644
index 0000000..1f1ee69
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateOperation.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.table.storage;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.ql.CompilationOpContext;
+import org.apache.hadoop.hive.ql.DriverContext;
+import org.apache.hadoop.hive.ql.ddl.DDLOperation;
+import org.apache.hadoop.hive.ql.ddl.DDLOperationContext;
+import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.OperatorFactory;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.tez.TezTask;
+import org.apache.hadoop.hive.ql.io.RCFileInputFormat;
+import org.apache.hadoop.hive.ql.io.merge.MergeFileTask;
+import org.apache.hadoop.hive.ql.io.merge.MergeFileWork;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.FileMergeDesc;
+import org.apache.hadoop.hive.ql.plan.ListBucketingCtx;
+import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+import org.apache.hadoop.hive.ql.plan.OrcFileMergeDesc;
+import org.apache.hadoop.hive.ql.plan.RCFileMergeDesc;
+import org.apache.hadoop.hive.ql.plan.TezWork;
+
+/**
+ * Operation process of concatenating the files of a table/partition.
+ */
+public class AlterTableConcatenateOperation extends DDLOperation {
+ private final AlterTableConcatenateDesc desc;
+
+ public AlterTableConcatenateOperation(DDLOperationContext context, AlterTableConcatenateDesc desc) {
+ super(context);
+ this.desc = desc;
+ }
+
+ @Override
+ public int execute() throws HiveException {
+ ListBucketingCtx lbCtx = desc.getLbCtx();
+ boolean lbatc = lbCtx == null ? false : lbCtx.isSkewedStoredAsDir();
+ int lbd = lbCtx == null ? 0 : lbCtx.calculateListBucketingLevel();
+
+ // merge work only needs input and output.
+ MergeFileWork mergeWork = new MergeFileWork(desc.getInputDir(), desc.getOutputDir(),
+ desc.getInputFormatClass().getName(), desc.getTableDesc());
+ LinkedHashMap<Path, ArrayList<String>> pathToAliases = new LinkedHashMap<>();
+ ArrayList<String> inputDirstr = new ArrayList<String>(1);
+ inputDirstr.add(desc.getInputDir().toString());
+ pathToAliases.put(desc.getInputDir().get(0), inputDirstr);
+ mergeWork.setPathToAliases(pathToAliases);
+ mergeWork.setListBucketingCtx(desc.getLbCtx());
+ mergeWork.resolveConcatenateMerge(context.getDb().getConf());
+ mergeWork.setMapperCannotSpanPartns(true);
+ mergeWork.setSourceTableInputFormat(desc.getInputFormatClass().getName());
+ final FileMergeDesc fmd;
+ if (desc.getInputFormatClass().equals(RCFileInputFormat.class)) {
+ fmd = new RCFileMergeDesc();
+ } else {
+ // safe to assume else is ORC as semantic analyzer will check for RC/ORC
+ fmd = new OrcFileMergeDesc();
+ }
+
+ fmd.setDpCtx(null);
+ fmd.setHasDynamicPartitions(false);
+ fmd.setListBucketingAlterTableConcatenate(lbatc);
+ fmd.setListBucketingDepth(lbd);
+ fmd.setOutputPath(desc.getOutputDir());
+
+ CompilationOpContext opContext = context.getDriverContext().getCtx().getOpContext();
+ Operator<? extends OperatorDesc> mergeOp = OperatorFactory.get(opContext, fmd);
+
+ LinkedHashMap<String, Operator<? extends OperatorDesc>> aliasToWork =
+ new LinkedHashMap<String, Operator<? extends OperatorDesc>>();
+ aliasToWork.put(desc.getInputDir().toString(), mergeOp);
+ mergeWork.setAliasToWork(aliasToWork);
+ DriverContext driverCxt = new DriverContext();
+ Task<?> task;
+ if (context.getConf().getVar(ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
+ TezWork tezWork = new TezWork(context.getQueryState().getQueryId(), context.getConf());
+ mergeWork.setName("File Merge");
+ tezWork.add(mergeWork);
+ task = new TezTask();
+ ((TezTask) task).setWork(tezWork);
+ } else {
+ task = new MergeFileTask();
+ ((MergeFileTask) task).setWork(mergeWork);
+ }
+
+ // initialize the task and execute
+ task.initialize(context.getQueryState(), context.getQueryPlan(), driverCxt, opContext);
+ Task<? extends Serializable> subtask = task;
+ int ret = task.execute(driverCxt);
+ if (subtask.getException() != null) {
+ context.getTask().setException(subtask.getException());
+ }
+ return ret;
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableIntoBucketsDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableIntoBucketsDesc.java
index 680f310..744769d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableIntoBucketsDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableIntoBucketsDesc.java
@@ -22,8 +22,8 @@ import java.util.Map;
import org.apache.hadoop.hive.ql.ddl.DDLTask2;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
@@ -42,7 +42,7 @@ public class AlterTableIntoBucketsDesc extends AbstractAlterTableDesc {
public AlterTableIntoBucketsDesc(String tableName, Map<String, String> partitionSpec, int numberOfBuckets)
throws SemanticException {
- super(AlterTableTypes.INTO_BUCKETS, tableName, partitionSpec, null, false, false, null);
+ super(AlterTableType.INTO_BUCKETS, tableName, partitionSpec, null, false, false, null);
this.numberOfBuckets = numberOfBuckets;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotClusteredDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotClusteredDesc.java
index a335d0d..fe4c413 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotClusteredDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotClusteredDesc.java
@@ -22,8 +22,8 @@ import java.util.Map;
import org.apache.hadoop.hive.ql.ddl.DDLTask2;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
@@ -39,7 +39,7 @@ public class AlterTableNotClusteredDesc extends AbstractAlterTableDesc {
}
public AlterTableNotClusteredDesc(String tableName, Map<String, String> partitionSpec) throws SemanticException {
- super(AlterTableTypes.NOT_CLUSTERED, tableName, partitionSpec, null, false, false, null);
+ super(AlterTableType.NOT_CLUSTERED, tableName, partitionSpec, null, false, false, null);
}
@Override
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSkewedDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSkewedDesc.java
index af67964..080dcfb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSkewedDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSkewedDesc.java
@@ -20,8 +20,8 @@ package org.apache.hadoop.hive.ql.ddl.table.storage;
import org.apache.hadoop.hive.ql.ddl.DDLTask2;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
@@ -37,7 +37,7 @@ public class AlterTableNotSkewedDesc extends AbstractAlterTableDesc {
}
public AlterTableNotSkewedDesc(String tableName) throws SemanticException {
- super(AlterTableTypes.NOT_SKEWED, tableName, null, null, false, false, null);
+ super(AlterTableType.NOT_SKEWED, tableName, null, null, false, false, null);
}
@Override
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSortedDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSortedDesc.java
index 11e8bf3..adb7c0f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSortedDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSortedDesc.java
@@ -22,8 +22,8 @@ import java.util.Map;
import org.apache.hadoop.hive.ql.ddl.DDLTask2;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
@@ -39,7 +39,7 @@ public class AlterTableNotSortedDesc extends AbstractAlterTableDesc {
}
public AlterTableNotSortedDesc(String tableName, Map<String, String> partitionSpec) throws SemanticException {
- super(AlterTableTypes.NOT_SORTED, tableName, partitionSpec, null, false, false, null);
+ super(AlterTableType.NOT_SORTED, tableName, partitionSpec, null, false, false, null);
}
@Override
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetFileFormatDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetFileFormatDesc.java
index 89bbb17..e2408d8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetFileFormatDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetFileFormatDesc.java
@@ -22,8 +22,8 @@ import java.util.Map;
import org.apache.hadoop.hive.ql.ddl.DDLTask2;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
@@ -44,7 +44,7 @@ public class AlterTableSetFileFormatDesc extends AbstractAlterTableDesc {
public AlterTableSetFileFormatDesc(String tableName, Map<String, String> partitionSpec, String inputFormat,
String outputFormat, String serdeName) throws SemanticException {
- super(AlterTableTypes.SET_FILE_FORMAT, tableName, partitionSpec, null, false, false, null);
+ super(AlterTableType.SET_FILE_FORMAT, tableName, partitionSpec, null, false, false, null);
this.inputFormat = inputFormat;
this.outputFormat = outputFormat;
this.serdeName = serdeName;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetLocationDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetLocationDesc.java
index c918bb9..573caab 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetLocationDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetLocationDesc.java
@@ -22,8 +22,8 @@ import java.util.Map;
import org.apache.hadoop.hive.ql.ddl.DDLTask2;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
@@ -42,7 +42,7 @@ public class AlterTableSetLocationDesc extends AbstractAlterTableDesc {
public AlterTableSetLocationDesc(String tableName, Map<String, String> partitionSpec, String location)
throws SemanticException {
- super(AlterTableTypes.SET_LOCATION, tableName, partitionSpec, null, false, false, null);
+ super(AlterTableType.ALTERLOCATION, tableName, partitionSpec, null, false, false, null);
this.location = location;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSerdeDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSerdeDesc.java
index 861139d..c245a2d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSerdeDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSerdeDesc.java
@@ -22,8 +22,8 @@ import java.util.Map;
import org.apache.hadoop.hive.ql.ddl.DDLTask2;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
@@ -42,7 +42,7 @@ public class AlterTableSetSerdeDesc extends AbstractAlterTableDesc {
public AlterTableSetSerdeDesc(String tableName, Map<String, String> partitionSpec, Map<String, String> props,
String serdeName) throws SemanticException {
- super(AlterTableTypes.SET_SERDE, tableName, partitionSpec, null, false, false, props);
+ super(AlterTableType.SET_SERDE, tableName, partitionSpec, null, false, false, props);
this.serdeName = serdeName;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSerdePropsDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSerdePropsDesc.java
index 381b94f..977ad93 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSerdePropsDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSerdePropsDesc.java
@@ -22,8 +22,8 @@ import java.util.Map;
import org.apache.hadoop.hive.ql.ddl.DDLTask2;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
@@ -40,7 +40,7 @@ public class AlterTableSetSerdePropsDesc extends AbstractAlterTableDesc {
public AlterTableSetSerdePropsDesc(String tableName, Map<String, String> partitionSpec, Map<String, String> props)
throws SemanticException {
- super(AlterTableTypes.SET_SERDE_PROPS, tableName, partitionSpec, null, false, false, props);
+ super(AlterTableType.SET_SERDE_PROPS, tableName, partitionSpec, null, false, false, props);
}
@Override
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSkewedLocationDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSkewedLocationDesc.java
index afe2b08..ddc8bb8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSkewedLocationDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSkewedLocationDesc.java
@@ -24,8 +24,8 @@ import java.util.stream.Collectors;
import org.apache.hadoop.hive.ql.ddl.DDLTask2;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
@@ -44,7 +44,7 @@ public class AlterTableSetSkewedLocationDesc extends AbstractAlterTableDesc {
public AlterTableSetSkewedLocationDesc(String tableName, Map<String, String> partitionSpec,
Map<List<String>, String> skewedLocations) throws SemanticException {
- super(AlterTableTypes.SET_SKEWED_LOCATION, tableName, partitionSpec, null, false, false, null);
+ super(AlterTableType.SET_SKEWED_LOCATION, tableName, partitionSpec, null, false, false, null);
this.skewedLocations = skewedLocations;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSkewedByDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSkewedByDesc.java
index 6a6f397..fa50cfd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSkewedByDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSkewedByDesc.java
@@ -23,8 +23,8 @@ import java.util.stream.Collectors;
import org.apache.hadoop.hive.ql.ddl.DDLTask2;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
@@ -45,7 +45,7 @@ public class AlterTableSkewedByDesc extends AbstractAlterTableDesc {
public AlterTableSkewedByDesc(String tableName, List<String> skewedColumnNames, List<List<String>> skewedColumnValues,
boolean isStoredAsDirectories) throws SemanticException {
- super(AlterTableTypes.SKEWED_BY, tableName, null, null, false, false, null);
+ super(AlterTableType.SKEWED_BY, tableName, null, null, false, false, null);
this.skewedColumnNames = skewedColumnNames;
this.skewedColumnValues = skewedColumnValues;
this.isStoredAsDirectories = isStoredAsDirectories;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSortedDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableUnarchiveDesc.java
similarity index 52%
copy from ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSortedDesc.java
copy to ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableUnarchiveDesc.java
index 11e8bf3..b75299c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSortedDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableUnarchiveDesc.java
@@ -20,30 +20,35 @@ package org.apache.hadoop.hive.ql.ddl.table.storage;
import java.util.Map;
+import org.apache.hadoop.hive.ql.ddl.DDLDesc;
import org.apache.hadoop.hive.ql.ddl.DDLTask2;
-import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc;
-import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
/**
- * DDL task description for ALTER TABLE ... NOT SORTED commands.
+ * DDL task description for ALTER TABLE ... UNARCHIVE [PARTITION ...] commands.
*/
-@Explain(displayName = "Not Sorted", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
-public class AlterTableNotSortedDesc extends AbstractAlterTableDesc {
- private static final long serialVersionUID = 1L;
-
+@Explain(displayName = "Unarchive", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+public class AlterTableUnarchiveDesc implements DDLDesc {
static {
- DDLTask2.registerOperation(AlterTableNotSortedDesc.class, AlterTableNotSortedOperation.class);
+ DDLTask2.registerOperation(AlterTableUnarchiveDesc.class, AlterTableUnarchiveOperation.class);
+ }
+
+ private final String tableName;
+ private final Map<String, String> partitionSpec;
+
+ public AlterTableUnarchiveDesc(String tableName, Map<String, String> partitionSpec) {
+ this.tableName = tableName;
+ this.partitionSpec = partitionSpec;
}
- public AlterTableNotSortedDesc(String tableName, Map<String, String> partitionSpec) throws SemanticException {
- super(AlterTableTypes.NOT_SORTED, tableName, partitionSpec, null, false, false, null);
+ @Explain(displayName = "table name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+ public String getTableName() {
+ return tableName;
}
- @Override
- public boolean mayNeedWriteId() {
- return false;
+ @Explain(displayName = "partition spec", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+ public Map<String, String> getPartitionSpec() {
+ return partitionSpec;
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableUnarchiveOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableUnarchiveOperation.java
new file mode 100644
index 0000000..248874f
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableUnarchiveOperation.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.table.storage;
+
+import org.apache.hadoop.hive.ql.ddl.DDLOperationContext;
+import org.apache.hadoop.hive.ql.exec.ArchiveUtils;
+import org.apache.hadoop.hive.ql.exec.ArchiveUtils.PartSpecInfo;
+import org.apache.hadoop.hive.ql.io.HdfsUtils;
+import org.apache.hadoop.hive.ql.ddl.DDLOperation;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.util.ToolRunner;
+
+import com.google.common.collect.ImmutableList;
+
+import static org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableArchiveUtils.ARCHIVE_NAME;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsShell;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+
+/**
+ * Operation process of truncating a table.
+ */
+public class AlterTableUnarchiveOperation extends DDLOperation {
+ private final AlterTableUnarchiveDesc desc;
+
+ public AlterTableUnarchiveOperation(DDLOperationContext context, AlterTableUnarchiveDesc desc) {
+ super(context);
+ this.desc = desc;
+ }
+
+ @Override
+ public int execute() throws HiveException, URISyntaxException {
+ Table table = context.getDb().getTable(desc.getTableName());
+ if (table.getTableType() != TableType.MANAGED_TABLE) {
+ throw new HiveException("UNARCHIVE can only be performed on managed tables");
+ }
+
+ Map<String, String> partitionSpec = desc.getPartitionSpec();
+ if (partitionSpec == null) {
+ throw new HiveException("UNARCHIVE is for partitions only");
+ }
+
+ PartSpecInfo partitionSpecInfo = PartSpecInfo.create(table, partitionSpec);
+ List<Partition> partitions = context.getDb().getPartitions(table, partitionSpec);
+
+ // The originalDir directory represents the original directory of the partitions' files. They now contain an
+ // archived version of those files eg. hdfs:/warehouse/myTable/ds=1/
+ Path originalDir = getOriginalDir(table, partitionSpecInfo, partitions);
+ Path intermediateArchivedDir = AlterTableArchiveUtils.getInterMediateDir(originalDir, context.getConf(),
+ ConfVars.METASTORE_INT_ARCHIVED);
+ Path intermediateExtractedDir = AlterTableArchiveUtils.getInterMediateDir(originalDir, context.getConf(),
+ ConfVars.METASTORE_INT_EXTRACTED);
+
+ boolean recovery = isRecovery(intermediateArchivedDir, intermediateExtractedDir);
+
+ for (Partition partition : partitions) {
+ checkArchiveProperty(partitionSpec.size(), recovery, partition);
+ }
+
+ // assume the archive is in the original dir, check if it exists
+ // The source directory is the directory containing all the files that should be in the partitions.
+ // e.g. har:/warehouse/myTable/ds=1/myTable.har/ - Note the har:/ scheme
+ Path archivePath = new Path(originalDir, ARCHIVE_NAME);
+ URI archiveUri = archivePath.toUri();
+ URI originalUri = ArchiveUtils.addSlash(originalDir.toUri());
+ ArchiveUtils.HarPathHelper harHelper = new ArchiveUtils.HarPathHelper(context.getConf(), archiveUri, originalUri);
+ URI sourceUri = harHelper.getHarUri(originalUri);
+ Path sourceDir = new Path(sourceUri.getScheme(), sourceUri.getAuthority(), sourceUri.getPath());
+
+ if (!HdfsUtils.pathExists(intermediateArchivedDir, context.getConf()) &&
+ !HdfsUtils.pathExists(archivePath, context.getConf())) {
+ throw new HiveException("Haven't found any archive where it should be");
+ }
+
+ Path tmpPath = context.getDriverContext().getCtx().getExternalTmpPath(originalDir);
+
+ FileSystem fs = null;
+ try {
+ fs = tmpPath.getFileSystem(context.getConf());
+ } catch (IOException e) {
+ throw new HiveException(e);
+ }
+
+ if (!recovery) {
+ extractArchiveToIntermediateDir(intermediateExtractedDir, sourceDir, tmpPath, fs);
+ }
+
+ // At this point, we know that the extracted files are in the intermediate extracted dir, or in the the original dir
+
+ moveOriginalDirToIntermediateDir(originalDir, intermediateArchivedDir, fs);
+
+ // If there is a failure from here to until when the metadata is changed, the partition will be empty or throw
+ // errors on read. If the original location exists here, then it must be the extracted files because in the previous
+ // step, we moved the previous original location (containing the archived files) to intermediateArchiveDir
+
+ moveIntermediateExtractedDirToOriginalParent(originalDir, intermediateExtractedDir, fs);
+
+ writeUnarchivationToMetastore(partitions);
+
+ // If a failure happens here, the intermediate archive files won't be
+ // deleted. The user will need to call unarchive again to clear those up.
+
+ deleteIntermediateArchivedDir(table, intermediateArchivedDir);
+
+ if (recovery) {
+ context.getConsole().printInfo("Recovery after UNARCHIVE succeeded");
+ }
+
+ return 0;
+ }
+
+ private Path getOriginalDir(Table table, PartSpecInfo partitionSpecInfo, List<Partition> partitions)
+ throws HiveException {
+ Path originalDir = null;
+
+ // when we have partial partitions specification we must assume partitions
+ // lie in standard place - if they were in custom locations putting
+ // them into one archive would involve mass amount of copying
+ // in full partition specification case we allow custom locations
+ // to keep backward compatibility
+ if (partitions.isEmpty()) {
+ throw new HiveException("No partition matches the specification");
+ } else if (partitionSpecInfo.values.size() != table.getPartCols().size()) {
+ // for partial specifications we need partitions to follow the scheme
+ for (Partition partition : partitions){
+ if (AlterTableArchiveUtils.partitionInCustomLocation(table, partition)) {
+ String message = String.format("UNARCHIVE cannot run for partition groups with custom locations like %s",
+ partition.getLocation());
+ throw new HiveException(message);
+ }
+ }
+ originalDir = partitionSpecInfo.createPath(table);
+ } else {
+ Partition partition = partitions.get(0);
+ if (ArchiveUtils.isArchived(partition)) {
+ originalDir = new Path(AlterTableArchiveUtils.getOriginalLocation(partition));
+ } else {
+ originalDir = new Path(partition.getLocation());
+ }
+ }
+ return originalDir;
+ }
+
+ private boolean isRecovery(Path intermediateArchivedDir, Path intermediateExtractedDir) throws HiveException {
+ if (HdfsUtils.pathExists(intermediateArchivedDir, context.getConf()) ||
+ HdfsUtils.pathExists(intermediateExtractedDir, context.getConf())) {
+ context.getConsole().printInfo("Starting recovery after failed UNARCHIVE");
+ return true;
+ }
+ return false;
+ }
+
+ private void checkArchiveProperty(int partitionSpecLevel, boolean recovery, Partition partition)
+ throws HiveException {
+ if (!ArchiveUtils.isArchived(partition) && !recovery) {
+ throw new HiveException("Partition " + partition.getName() + " is not archived.");
+ }
+ int archiveLevel = ArchiveUtils.getArchivingLevel(partition);
+ if (partitionSpecLevel > archiveLevel) {
+ throw new HiveException("Partition " + partition.getName() + " is archived at level " + archiveLevel +
+ ", and given partspec only has " + partitionSpecLevel + " specs.");
+ }
+ }
+
+ private void extractArchiveToIntermediateDir(Path intermediateExtractedDir, Path sourceDir, Path tmpPath,
+ FileSystem fs) throws HiveException {
+ try {
+ // Copy the files out of the archive into the temporary directory
+ String copySource = sourceDir.toString();
+ String copyDest = tmpPath.toString();
+ List<String> args = ImmutableList.of("-cp", copySource, copyDest);
+
+ context.getConsole().printInfo("Copying " + copySource + " to " + copyDest);
+ FileSystem srcFs = FileSystem.get(sourceDir.toUri(), context.getConf());
+ srcFs.initialize(sourceDir.toUri(), context.getConf());
+
+ FsShell fss = new FsShell(context.getConf());
+ int ret = 0;
+ try {
+ ret = ToolRunner.run(fss, args.toArray(new String[0]));
+ } catch (Exception e) {
+ throw new HiveException(e);
+ }
+
+ if (ret != 0) {
+ throw new HiveException("Error while copying files from archive, return code=" + ret);
+ } else {
+ context.getConsole().printInfo("Successfully Copied " + copySource + " to " + copyDest);
+ }
+
+ context.getConsole().printInfo("Moving " + tmpPath + " to " + intermediateExtractedDir);
+ if (fs.exists(intermediateExtractedDir)) {
+ throw new HiveException("Invalid state: the intermediate extracted directory already exists.");
+ }
+ fs.rename(tmpPath, intermediateExtractedDir);
+ } catch (Exception e) {
+ throw new HiveException(e);
+ }
+ }
+
+ private void moveOriginalDirToIntermediateDir(Path originalDir, Path intermediateArchivedDir, FileSystem fs)
+ throws HiveException {
+ if (!HdfsUtils.pathExists(intermediateArchivedDir, context.getConf())) {
+ try {
+ context.getConsole().printInfo("Moving " + originalDir + " to " + intermediateArchivedDir);
+ fs.rename(originalDir, intermediateArchivedDir);
+ } catch (IOException e) {
+ throw new HiveException(e);
+ }
+ } else {
+ context.getConsole().printInfo(intermediateArchivedDir + " already exists. " +
+ "Assuming it contains the archived version of the partition");
+ }
+ }
+
+ private void moveIntermediateExtractedDirToOriginalParent(Path originalDir, Path intermediateExtractedDir,
+ FileSystem fs) throws HiveException {
+ if (!HdfsUtils.pathExists(originalDir, context.getConf())) {
+ try {
+ context.getConsole().printInfo("Moving " + intermediateExtractedDir + " to " + originalDir);
+ fs.rename(intermediateExtractedDir, originalDir);
+ } catch (IOException e) {
+ throw new HiveException(e);
+ }
+ } else {
+ context.getConsole().printInfo(originalDir + " already exists. " +
+ "Assuming it contains the extracted files in the partition");
+ }
+ }
+
+ private void writeUnarchivationToMetastore(List<Partition> partitions) throws HiveException {
+ for (Partition partition : partitions) {
+ setUnArchived(partition);
+ try {
+ // TODO: use getDb().alterPartition with catalog name
+ context.getDb().alterPartition(desc.getTableName(), partition, null, true);
+ } catch (InvalidOperationException e) {
+ throw new HiveException(e);
+ }
+ }
+ }
+
+ /**
+ * Sets the appropriate attributes in the supplied Partition object to mark it as not archived.
+ * Note that the metastore is not touched - a separate call to alter_partition is needed.
+ *
+ * @param partition - the partition to modify
+ */
+ private void setUnArchived(Partition partition) {
+ assert(ArchiveUtils.isArchived(partition));
+ String parentDir = AlterTableArchiveUtils.getOriginalLocation(partition);
+ AlterTableArchiveUtils.setIsArchived(partition, false, 0);
+ AlterTableArchiveUtils.setOriginalLocation(partition, null);
+ assert(parentDir != null);
+ partition.setLocation(parentDir);
+ }
+
+ private void deleteIntermediateArchivedDir(Table table, Path intermediateArchivedDir) throws HiveException {
+ if (HdfsUtils.pathExists(intermediateArchivedDir, context.getConf())) {
+ AlterTableArchiveUtils.deleteDir(intermediateArchivedDir, context.getDb().getDatabase(table.getDbName()),
+ context.getConf());
+ }
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index b66c308..89dca2e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -19,88 +19,37 @@
package org.apache.hadoop.hive.ql.exec;
import java.io.DataOutputStream;
-import java.io.IOException;
import java.io.Serializable;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
import java.util.Map;
import java.util.Set;
-import com.google.common.collect.Lists;
-import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.DefaultHiveMetaHook;
import org.apache.hadoop.hive.metastore.HiveMetaHook;
import org.apache.hadoop.hive.metastore.Msck;
import org.apache.hadoop.hive.metastore.MsckInfo;
-import org.apache.hadoop.hive.metastore.TableType;
-import org.apache.hadoop.hive.metastore.Warehouse;
-import org.apache.hadoop.hive.metastore.api.CompactionResponse;
import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
-import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
-import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.txn.TxnStore;
-import org.apache.hadoop.hive.ql.CompilationOpContext;
import org.apache.hadoop.hive.ql.DriverContext;
-import org.apache.hadoop.hive.ql.ErrorMsg;
-import org.apache.hadoop.hive.ql.QueryPlan;
-import org.apache.hadoop.hive.ql.QueryState;
-import org.apache.hadoop.hive.ql.exec.ArchiveUtils.PartSpecInfo;
-import org.apache.hadoop.hive.ql.exec.tez.TezTask;
-import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
-import org.apache.hadoop.hive.ql.io.AcidUtils;
-import org.apache.hadoop.hive.ql.io.RCFileInputFormat;
-import org.apache.hadoop.hive.ql.io.merge.MergeFileTask;
-import org.apache.hadoop.hive.ql.io.merge.MergeFileWork;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.Partition;
-import org.apache.hadoop.hive.ql.metadata.PartitionIterable;
-import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.parse.AlterTablePartMergeFilesDesc;
-import org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState;
-import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.parse.repl.dump.Utils;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
-import org.apache.hadoop.hive.ql.plan.AlterTableSimpleDesc;
import org.apache.hadoop.hive.ql.plan.CacheMetadataDesc;
import org.apache.hadoop.hive.ql.plan.DDLWork;
-import org.apache.hadoop.hive.ql.plan.FileMergeDesc;
import org.apache.hadoop.hive.ql.plan.InsertCommitHookDesc;
-import org.apache.hadoop.hive.ql.plan.ListBucketingCtx;
-import org.apache.hadoop.hive.ql.plan.LoadMultiFilesDesc;
-import org.apache.hadoop.hive.ql.plan.MoveWork;
import org.apache.hadoop.hive.ql.plan.MsckDesc;
-import org.apache.hadoop.hive.ql.plan.OperatorDesc;
-import org.apache.hadoop.hive.ql.plan.OrcFileMergeDesc;
-import org.apache.hadoop.hive.ql.plan.RCFileMergeDesc;
import org.apache.hadoop.hive.ql.plan.ReplRemoveFirstIncLoadPendFlagDesc;
import org.apache.hadoop.hive.ql.plan.ShowConfDesc;
-import org.apache.hadoop.hive.ql.plan.TezWork;
import org.apache.hadoop.hive.ql.plan.api.StageType;
import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.Deserializer;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.tools.HadoopArchives;
-import org.apache.hadoop.util.ToolRunner;
import org.apache.hive.common.util.ReflectionUtil;
import org.apache.thrift.TException;
import org.slf4j.Logger;
@@ -117,12 +66,6 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
private static final int separator = Utilities.tabCode;
private static final int terminator = Utilities.newLineCode;
- // These are suffixes attached to intermediate directory names used in the
- // archiving / un-archiving process.
- private static String INTERMEDIATE_ARCHIVED_DIR_SUFFIX;
- private static String INTERMEDIATE_ORIGINAL_DIR_SUFFIX;
- private static String INTERMEDIATE_EXTRACTED_DIR_SUFFIX;
-
@Override
public boolean requireLock() {
return this.work != null && this.work.getNeedLock();
@@ -133,21 +76,6 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
}
@Override
- public void initialize(QueryState queryState, QueryPlan queryPlan, DriverContext ctx,
- CompilationOpContext opContext) {
- super.initialize(queryState, queryPlan, ctx, opContext);
-
- // Pick the formatter to use to display the results. Either the
- // normal human readable output or a json object.
- INTERMEDIATE_ARCHIVED_DIR_SUFFIX =
- HiveConf.getVar(conf, ConfVars.METASTORE_INT_ARCHIVED);
- INTERMEDIATE_ORIGINAL_DIR_SUFFIX =
- HiveConf.getVar(conf, ConfVars.METASTORE_INT_ORIGINAL);
- INTERMEDIATE_EXTRACTED_DIR_SUFFIX =
- HiveConf.getVar(conf, ConfVars.METASTORE_INT_EXTRACTED);
- }
-
- @Override
public int execute(DriverContext driverContext) {
if (driverContext.getCtx().getExplainAnalyze() == AnalyzeState.RUNNING) {
return 0;
@@ -158,30 +86,6 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
try {
db = Hive.get(conf);
- AlterTableDesc alterTbl = work.getAlterTblDesc();
- if (alterTbl != null) {
- if (!allowOperationInReplicationScope(db, alterTbl.getOldName(), null, alterTbl.getReplicationSpec())) {
- // no alter, the table is missing either due to drop/rename which follows the alter.
- // or the existing table is newer than our update.
- LOG.debug("DDLTask: Alter Table is skipped as table {} is newer than update", alterTbl.getOldName());
- return 0;
- }
- return alterTable(db, alterTbl);
- }
-
- AlterTableSimpleDesc simpleDesc = work.getAlterTblSimpleDesc();
- if (simpleDesc != null) {
- if (simpleDesc.getType() == AlterTableTypes.TOUCH) {
- return touch(db, simpleDesc);
- } else if (simpleDesc.getType() == AlterTableTypes.ARCHIVE) {
- return archive(db, simpleDesc, driverContext);
- } else if (simpleDesc.getType() == AlterTableTypes.UNARCHIVE) {
- return unarchive(db, simpleDesc);
- } else if (simpleDesc.getType() == AlterTableTypes.COMPACT) {
- return compact(db, simpleDesc);
- }
- }
-
MsckDesc msckDesc = work.getMsckDesc();
if (msckDesc != null) {
return msck(db, msckDesc);
@@ -192,11 +96,6 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
return showConf(db, showConf);
}
- AlterTablePartMergeFilesDesc mergeFilesDesc = work.getMergeFilesDesc();
- if (mergeFilesDesc != null) {
- return mergeFiles(db, mergeFilesDesc, driverContext);
- }
-
CacheMetadataDesc cacheMetadataDesc = work.getCacheMetadataDesc();
if (cacheMetadataDesc != null) {
return cacheMetadata(db, cacheMetadataDesc);
@@ -288,771 +187,6 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
}
/**
- * First, make sure the source table/partition is not
- * archived/indexes/non-rcfile. If either of these is true, throw an
- * exception.
- *
- * The way how it does the merge is to create a BlockMergeTask from the
- * mergeFilesDesc.
- *
- * @param db
- * @param mergeFilesDesc
- * @return
- * @throws HiveException
- */
- private int mergeFiles(Hive db, AlterTablePartMergeFilesDesc mergeFilesDesc,
- DriverContext driverContext) throws HiveException {
- ListBucketingCtx lbCtx = mergeFilesDesc.getLbCtx();
- boolean lbatc = lbCtx == null ? false : lbCtx.isSkewedStoredAsDir();
- int lbd = lbCtx == null ? 0 : lbCtx.calculateListBucketingLevel();
-
- // merge work only needs input and output.
- MergeFileWork mergeWork = new MergeFileWork(mergeFilesDesc.getInputDir(),
- mergeFilesDesc.getOutputDir(), mergeFilesDesc.getInputFormatClass().getName(),
- mergeFilesDesc.getTableDesc());
- LinkedHashMap<Path, ArrayList<String>> pathToAliases = new LinkedHashMap<>();
- ArrayList<String> inputDirstr = new ArrayList<String>(1);
- inputDirstr.add(mergeFilesDesc.getInputDir().toString());
- pathToAliases.put(mergeFilesDesc.getInputDir().get(0), inputDirstr);
- mergeWork.setPathToAliases(pathToAliases);
- mergeWork.setListBucketingCtx(mergeFilesDesc.getLbCtx());
- mergeWork.resolveConcatenateMerge(db.getConf());
- mergeWork.setMapperCannotSpanPartns(true);
- mergeWork.setSourceTableInputFormat(mergeFilesDesc.getInputFormatClass().getName());
- final FileMergeDesc fmd;
- if (mergeFilesDesc.getInputFormatClass().equals(RCFileInputFormat.class)) {
- fmd = new RCFileMergeDesc();
- } else {
- // safe to assume else is ORC as semantic analyzer will check for RC/ORC
- fmd = new OrcFileMergeDesc();
- }
-
- fmd.setDpCtx(null);
- fmd.setHasDynamicPartitions(false);
- fmd.setListBucketingAlterTableConcatenate(lbatc);
- fmd.setListBucketingDepth(lbd);
- fmd.setOutputPath(mergeFilesDesc.getOutputDir());
-
- CompilationOpContext opContext = driverContext.getCtx().getOpContext();
- Operator<? extends OperatorDesc> mergeOp = OperatorFactory.get(opContext, fmd);
-
- LinkedHashMap<String, Operator<? extends OperatorDesc>> aliasToWork =
- new LinkedHashMap<String, Operator<? extends OperatorDesc>>();
- aliasToWork.put(mergeFilesDesc.getInputDir().toString(), mergeOp);
- mergeWork.setAliasToWork(aliasToWork);
- DriverContext driverCxt = new DriverContext();
- Task<?> task;
- if (conf.getVar(ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
- TezWork tezWork = new TezWork(queryState.getQueryId(), conf);
- mergeWork.setName("File Merge");
- tezWork.add(mergeWork);
- task = new TezTask();
- ((TezTask) task).setWork(tezWork);
- } else {
- task = new MergeFileTask();
- ((MergeFileTask) task).setWork(mergeWork);
- }
-
- // initialize the task and execute
- task.initialize(queryState, getQueryPlan(), driverCxt, opContext);
- Task<? extends Serializable> subtask = task;
- int ret = task.execute(driverCxt);
- if (subtask.getException() != null) {
- setException(subtask.getException());
- }
- return ret;
- }
-
- /**
- * Rewrite the partition's metadata and force the pre/post execute hooks to
- * be fired.
- *
- * @param db
- * @param touchDesc
- * @return
- * @throws HiveException
- */
- private int touch(Hive db, AlterTableSimpleDesc touchDesc)
- throws HiveException {
- // TODO: catalog
- Table tbl = db.getTable(touchDesc.getTableName());
- EnvironmentContext environmentContext = new EnvironmentContext();
- environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE);
-
- if (touchDesc.getPartSpec() == null) {
- db.alterTable(tbl, false, environmentContext, true);
- work.getInputs().add(new ReadEntity(tbl));
- addIfAbsentByName(new WriteEntity(tbl, WriteEntity.WriteType.DDL_NO_LOCK));
- } else {
- Partition part = db.getPartition(tbl, touchDesc.getPartSpec(), false);
- if (part == null) {
- throw new HiveException("Specified partition does not exist");
- }
- try {
- db.alterPartition(tbl.getCatalogName(), tbl.getDbName(), tbl.getTableName(),
- part, environmentContext, true);
- } catch (InvalidOperationException e) {
- throw new HiveException(e);
- }
- work.getInputs().add(new ReadEntity(part));
- addIfAbsentByName(new WriteEntity(part, WriteEntity.WriteType.DDL_NO_LOCK));
- }
- return 0;
- }
-
- /**
- * Sets archiving flag locally; it has to be pushed into metastore
- * @param p partition to set flag
- * @param state desired state of IS_ARCHIVED flag
- * @param level desired level for state == true, anything for false
- */
- private void setIsArchived(Partition p, boolean state, int level) {
- Map<String, String> params = p.getParameters();
- if (state) {
- params.put(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.IS_ARCHIVED,
- "true");
- params.put(ArchiveUtils.ARCHIVING_LEVEL, Integer
- .toString(level));
- } else {
- params.remove(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.IS_ARCHIVED);
- params.remove(ArchiveUtils.ARCHIVING_LEVEL);
- }
- }
-
- /**
- * Returns original partition of archived partition, null for unarchived one
- */
- private String getOriginalLocation(Partition p) {
- Map<String, String> params = p.getParameters();
- return params.get(
- org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ORIGINAL_LOCATION);
- }
-
- /**
- * Sets original location of partition which is to be archived
- */
- private void setOriginalLocation(Partition p, String loc) {
- Map<String, String> params = p.getParameters();
- if (loc == null) {
- params.remove(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ORIGINAL_LOCATION);
- } else {
- params.put(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ORIGINAL_LOCATION, loc);
- }
- }
-
- /**
- * Sets the appropriate attributes in the supplied Partition object to mark
- * it as archived. Note that the metastore is not touched - a separate
- * call to alter_partition is needed.
- *
- * @param p - the partition object to modify
- * @param harPath - new location of partition (har schema URI)
- */
- private void setArchived(Partition p, Path harPath, int level) {
- assert(ArchiveUtils.isArchived(p) == false);
- setIsArchived(p, true, level);
- setOriginalLocation(p, p.getLocation());
- p.setLocation(harPath.toString());
- }
-
- /**
- * Sets the appropriate attributes in the supplied Partition object to mark
- * it as not archived. Note that the metastore is not touched - a separate
- * call to alter_partition is needed.
- *
- * @param p - the partition to modify
- */
- private void setUnArchived(Partition p) {
- assert(ArchiveUtils.isArchived(p) == true);
- String parentDir = getOriginalLocation(p);
- setIsArchived(p, false, 0);
- setOriginalLocation(p, null);
- assert(parentDir != null);
- p.setLocation(parentDir);
- }
-
- private boolean pathExists(Path p) throws HiveException {
- try {
- FileSystem fs = p.getFileSystem(conf);
- return fs.exists(p);
- } catch (IOException e) {
- throw new HiveException(e);
- }
- }
-
- private void moveDir(FileSystem fs, Path from, Path to) throws HiveException {
- try {
- if (!fs.rename(from, to)) {
- throw new HiveException("Moving " + from + " to " + to + " failed!");
- }
- } catch (IOException e) {
- throw new HiveException(e);
- }
- }
-
- private void deleteDir(Path dir, Database db) throws HiveException {
- try {
- Warehouse wh = new Warehouse(conf);
- wh.deleteDir(dir, true, db);
- } catch (MetaException e) {
- throw new HiveException(e);
- }
- }
-
- /**
- * Checks in partition is in custom (not-standard) location.
- * @param tbl - table in which partition is
- * @param p - partition
- * @return true if partition location is custom, false if it is standard
- */
- boolean partitionInCustomLocation(Table tbl, Partition p)
- throws HiveException {
- String subdir = null;
- try {
- subdir = Warehouse.makePartName(tbl.getPartCols(), p.getValues());
- } catch (MetaException e) {
- throw new HiveException("Unable to get partition's directory", e);
- }
- Path tableDir = tbl.getDataLocation();
- if(tableDir == null) {
- throw new HiveException("Table has no location set");
- }
-
- String standardLocation = (new Path(tableDir, subdir)).toString();
- if(ArchiveUtils.isArchived(p)) {
- return !getOriginalLocation(p).equals(standardLocation);
- } else {
- return !p.getLocation().equals(standardLocation);
- }
- }
-
- private int archive(Hive db, AlterTableSimpleDesc simpleDesc,
- DriverContext driverContext)
- throws HiveException {
-
- Table tbl = db.getTable(simpleDesc.getTableName());
-
- if (tbl.getTableType() != TableType.MANAGED_TABLE) {
- throw new HiveException("ARCHIVE can only be performed on managed tables");
- }
-
- Map<String, String> partSpec = simpleDesc.getPartSpec();
- PartSpecInfo partSpecInfo = PartSpecInfo.create(tbl, partSpec);
- List<Partition> partitions = db.getPartitions(tbl, partSpec);
-
- Path originalDir = null;
-
- // when we have partial partitions specification we must assume partitions
- // lie in standard place - if they were in custom locations putting
- // them into one archive would involve mass amount of copying
- // in full partition specification case we allow custom locations
- // to keep backward compatibility
- if (partitions.isEmpty()) {
- throw new HiveException("No partition matches the specification");
- } else if(partSpecInfo.values.size() != tbl.getPartCols().size()) {
- // for partial specifications we need partitions to follow the scheme
- for(Partition p: partitions){
- if(partitionInCustomLocation(tbl, p)) {
- String message = String.format("ARCHIVE cannot run for partition " +
- "groups with custom locations like %s", p.getLocation());
- throw new HiveException(message);
- }
- }
- originalDir = partSpecInfo.createPath(tbl);
- } else {
- Partition p = partitions.get(0);
- // partition can be archived if during recovery
- if(ArchiveUtils.isArchived(p)) {
- originalDir = new Path(getOriginalLocation(p));
- } else {
- originalDir = p.getDataLocation();
- }
- }
-
- Path intermediateArchivedDir = new Path(originalDir.getParent(),
- originalDir.getName() + INTERMEDIATE_ARCHIVED_DIR_SUFFIX);
- Path intermediateOriginalDir = new Path(originalDir.getParent(),
- originalDir.getName() + INTERMEDIATE_ORIGINAL_DIR_SUFFIX);
-
- console.printInfo("intermediate.archived is " + intermediateArchivedDir.toString());
- console.printInfo("intermediate.original is " + intermediateOriginalDir.toString());
-
- String archiveName = "data.har";
- FileSystem fs = null;
- try {
- fs = originalDir.getFileSystem(conf);
- } catch (IOException e) {
- throw new HiveException(e);
- }
-
- URI archiveUri = (new Path(originalDir, archiveName)).toUri();
- URI originalUri = ArchiveUtils.addSlash(originalDir.toUri());
- ArchiveUtils.HarPathHelper harHelper = new ArchiveUtils.HarPathHelper(
- conf, archiveUri, originalUri);
-
- // we checked if partitions matching specification are marked as archived
- // in the metadata; if they are and their levels are the same as we would
- // set it later it means previous run failed and we have to do the recovery;
- // if they are different, we throw an error
- for(Partition p: partitions) {
- if(ArchiveUtils.isArchived(p)) {
- if(ArchiveUtils.getArchivingLevel(p) != partSpecInfo.values.size()) {
- String name = ArchiveUtils.getPartialName(p, ArchiveUtils.getArchivingLevel(p));
- String m = String.format("Conflict with existing archive %s", name);
- throw new HiveException(m);
- } else {
- throw new HiveException("Partition(s) already archived");
- }
- }
- }
-
- boolean recovery = false;
- if (pathExists(intermediateArchivedDir)
- || pathExists(intermediateOriginalDir)) {
- recovery = true;
- console.printInfo("Starting recovery after failed ARCHIVE");
- }
-
- // The following steps seem roundabout, but they are meant to aid in
- // recovery if a failure occurs and to keep a consistent state in the FS
-
- // Steps:
- // 1. Create the archive in a temporary folder
- // 2. Move the archive dir to an intermediate dir that is in at the same
- // dir as the original partition dir. Call the new dir
- // intermediate-archive.
- // 3. Rename the original partition dir to an intermediate dir. Call the
- // renamed dir intermediate-original
- // 4. Rename intermediate-archive to the original partition dir
- // 5. Change the metadata
- // 6. Delete the original partition files in intermediate-original
-
- // The original partition files are deleted after the metadata change
- // because the presence of those files are used to indicate whether
- // the original partition directory contains archived or unarchived files.
-
- // Create an archived version of the partition in a directory ending in
- // ARCHIVE_INTERMEDIATE_DIR_SUFFIX that's the same level as the partition,
- // if it does not already exist. If it does exist, we assume the dir is good
- // to use as the move operation that created it is atomic.
- if (!pathExists(intermediateArchivedDir) &&
- !pathExists(intermediateOriginalDir)) {
-
- // First create the archive in a tmp dir so that if the job fails, the
- // bad files don't pollute the filesystem
- Path tmpPath = new Path(driverContext.getCtx()
- .getExternalTmpPath(originalDir), "partlevel");
-
- console.printInfo("Creating " + archiveName +
- " for " + originalDir.toString());
- console.printInfo("in " + tmpPath);
- console.printInfo("Please wait... (this may take a while)");
-
- // Create the Hadoop archive
- int ret=0;
- try {
- int maxJobNameLen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH);
- String jobname = String.format("Archiving %s@%s",
- tbl.getTableName(), partSpecInfo.getName());
- jobname = Utilities.abbreviate(jobname, maxJobNameLen - 6);
- conf.set(MRJobConfig.JOB_NAME, jobname);
- HadoopArchives har = new HadoopArchives(conf);
- List<String> args = new ArrayList<String>();
-
- args.add("-archiveName");
- args.add(archiveName);
- args.add("-p");
- args.add(originalDir.toString());
- args.add(tmpPath.toString());
-
- ret = ToolRunner.run(har, args.toArray(new String[0]));
- } catch (Exception e) {
- throw new HiveException(e);
- }
- if (ret != 0) {
- throw new HiveException("Error while creating HAR");
- }
-
- // Move from the tmp dir to an intermediate directory, in the same level as
- // the partition directory. e.g. .../hr=12-intermediate-archived
- try {
- console.printInfo("Moving " + tmpPath + " to " + intermediateArchivedDir);
- if (pathExists(intermediateArchivedDir)) {
- throw new HiveException("The intermediate archive directory already exists.");
- }
- fs.rename(tmpPath, intermediateArchivedDir);
- } catch (IOException e) {
- throw new HiveException("Error while moving tmp directory");
- }
- } else {
- if (pathExists(intermediateArchivedDir)) {
- console.printInfo("Intermediate archive directory " + intermediateArchivedDir +
- " already exists. Assuming it contains an archived version of the partition");
- }
- }
-
- // If we get to here, we know that we've archived the partition files, but
- // they may be in the original partition location, or in the intermediate
- // original dir.
-
- // Move the original parent directory to the intermediate original directory
- // if the move hasn't been made already
- if (!pathExists(intermediateOriginalDir)) {
- console.printInfo("Moving " + originalDir + " to " +
- intermediateOriginalDir);
- moveDir(fs, originalDir, intermediateOriginalDir);
- } else {
- console.printInfo(intermediateOriginalDir + " already exists. " +
- "Assuming it contains the original files in the partition");
- }
-
- // If there's a failure from here to when the metadata is updated,
- // there will be no data in the partition, or an error while trying to read
- // the partition (if the archive files have been moved to the original
- // partition directory.) But re-running the archive command will allow
- // recovery
-
- // Move the intermediate archived directory to the original parent directory
- if (!pathExists(originalDir)) {
- console.printInfo("Moving " + intermediateArchivedDir + " to " +
- originalDir);
- moveDir(fs, intermediateArchivedDir, originalDir);
- } else {
- console.printInfo(originalDir + " already exists. " +
- "Assuming it contains the archived version of the partition");
- }
-
- // Record this change in the metastore
- try {
- for(Partition p: partitions) {
- URI originalPartitionUri = ArchiveUtils.addSlash(p.getDataLocation().toUri());
- URI harPartitionDir = harHelper.getHarUri(originalPartitionUri);
- StringBuilder authority = new StringBuilder();
- if(harPartitionDir.getUserInfo() != null) {
- authority.append(harPartitionDir.getUserInfo()).append("@");
- }
- authority.append(harPartitionDir.getHost());
- if(harPartitionDir.getPort() != -1) {
- authority.append(":").append(harPartitionDir.getPort());
- }
- Path harPath = new Path(harPartitionDir.getScheme(),
- authority.toString(),
- harPartitionDir.getPath()); // make in Path to ensure no slash at the end
- setArchived(p, harPath, partSpecInfo.values.size());
- // TODO: catalog
- db.alterPartition(simpleDesc.getTableName(), p, null, true);
- }
- } catch (Exception e) {
- throw new HiveException("Unable to change the partition info for HAR", e);
- }
-
- // If a failure occurs here, the directory containing the original files
- // will not be deleted. The user will run ARCHIVE again to clear this up
- if(pathExists(intermediateOriginalDir)) {
- deleteDir(intermediateOriginalDir, db.getDatabase(tbl.getDbName()));
- }
-
- if(recovery) {
- console.printInfo("Recovery after ARCHIVE succeeded");
- }
-
- return 0;
- }
-
- private int unarchive(Hive db, AlterTableSimpleDesc simpleDesc)
- throws HiveException, URISyntaxException {
-
- Table tbl = db.getTable(simpleDesc.getTableName());
-
- // Means user specified a table, not a partition
- if (simpleDesc.getPartSpec() == null) {
- throw new HiveException("UNARCHIVE is for partitions only");
- }
-
- if (tbl.getTableType() != TableType.MANAGED_TABLE) {
- throw new HiveException("UNARCHIVE can only be performed on managed tables");
- }
-
- Map<String, String> partSpec = simpleDesc.getPartSpec();
- PartSpecInfo partSpecInfo = PartSpecInfo.create(tbl, partSpec);
- List<Partition> partitions = db.getPartitions(tbl, partSpec);
-
- int partSpecLevel = partSpec.size();
-
- Path originalDir = null;
-
- // when we have partial partitions specification we must assume partitions
- // lie in standard place - if they were in custom locations putting
- // them into one archive would involve mass amount of copying
- // in full partition specification case we allow custom locations
- // to keep backward compatibility
- if (partitions.isEmpty()) {
- throw new HiveException("No partition matches the specification");
- } else if(partSpecInfo.values.size() != tbl.getPartCols().size()) {
- // for partial specifications we need partitions to follow the scheme
- for(Partition p: partitions){
- if(partitionInCustomLocation(tbl, p)) {
- String message = String.format("UNARCHIVE cannot run for partition " +
- "groups with custom locations like %s", p.getLocation());
- throw new HiveException(message);
- }
- }
- originalDir = partSpecInfo.createPath(tbl);
- } else {
- Partition p = partitions.get(0);
- if(ArchiveUtils.isArchived(p)) {
- originalDir = new Path(getOriginalLocation(p));
- } else {
- originalDir = new Path(p.getLocation());
- }
- }
-
- URI originalUri = ArchiveUtils.addSlash(originalDir.toUri());
- Path intermediateArchivedDir = new Path(originalDir.getParent(),
- originalDir.getName() + INTERMEDIATE_ARCHIVED_DIR_SUFFIX);
- Path intermediateExtractedDir = new Path(originalDir.getParent(),
- originalDir.getName() + INTERMEDIATE_EXTRACTED_DIR_SUFFIX);
- boolean recovery = false;
- if(pathExists(intermediateArchivedDir) || pathExists(intermediateExtractedDir)) {
- recovery = true;
- console.printInfo("Starting recovery after failed UNARCHIVE");
- }
-
- for(Partition p: partitions) {
- checkArchiveProperty(partSpecLevel, recovery, p);
- }
-
- String archiveName = "data.har";
- FileSystem fs = null;
- try {
- fs = originalDir.getFileSystem(conf);
- } catch (IOException e) {
- throw new HiveException(e);
- }
-
- // assume the archive is in the original dir, check if it exists
- Path archivePath = new Path(originalDir, archiveName);
- URI archiveUri = archivePath.toUri();
- ArchiveUtils.HarPathHelper harHelper = new ArchiveUtils.HarPathHelper(conf,
- archiveUri, originalUri);
- URI sourceUri = harHelper.getHarUri(originalUri);
- Path sourceDir = new Path(sourceUri.getScheme(), sourceUri.getAuthority(), sourceUri.getPath());
-
- if(!pathExists(intermediateArchivedDir) && !pathExists(archivePath)) {
- throw new HiveException("Haven't found any archive where it should be");
- }
-
- Path tmpPath = driverContext.getCtx().getExternalTmpPath(originalDir);
-
- try {
- fs = tmpPath.getFileSystem(conf);
- } catch (IOException e) {
- throw new HiveException(e);
- }
-
- // Clarification of terms:
- // - The originalDir directory represents the original directory of the
- // partitions' files. They now contain an archived version of those files
- // eg. hdfs:/warehouse/myTable/ds=1/
- // - The source directory is the directory containing all the files that
- // should be in the partitions. e.g. har:/warehouse/myTable/ds=1/myTable.har/
- // Note the har:/ scheme
-
- // Steps:
- // 1. Extract the archive in a temporary folder
- // 2. Move the archive dir to an intermediate dir that is in at the same
- // dir as originalLocation. Call the new dir intermediate-extracted.
- // 3. Rename the original partitions dir to an intermediate dir. Call the
- // renamed dir intermediate-archive
- // 4. Rename intermediate-extracted to the original partitions dir
- // 5. Change the metadata
- // 6. Delete the archived partitions files in intermediate-archive
-
- if (!pathExists(intermediateExtractedDir) &&
- !pathExists(intermediateArchivedDir)) {
- try {
-
- // Copy the files out of the archive into the temporary directory
- String copySource = sourceDir.toString();
- String copyDest = tmpPath.toString();
- List<String> args = new ArrayList<String>();
- args.add("-cp");
- args.add(copySource);
- args.add(copyDest);
-
- console.printInfo("Copying " + copySource + " to " + copyDest);
- FileSystem srcFs = FileSystem.get(sourceDir.toUri(), conf);
- srcFs.initialize(sourceDir.toUri(), conf);
-
- FsShell fss = new FsShell(conf);
- int ret = 0;
- try {
- ret = ToolRunner.run(fss, args.toArray(new String[0]));
- } catch (Exception e) {
- throw new HiveException(e);
- }
-
- if (ret != 0) {
- throw new HiveException("Error while copying files from archive, return code=" + ret);
- } else {
- console.printInfo("Successfully Copied " + copySource + " to " + copyDest);
- }
-
- console.printInfo("Moving " + tmpPath + " to " + intermediateExtractedDir);
- if (fs.exists(intermediateExtractedDir)) {
- throw new HiveException("Invalid state: the intermediate extracted " +
- "directory already exists.");
- }
- fs.rename(tmpPath, intermediateExtractedDir);
- } catch (Exception e) {
- throw new HiveException(e);
- }
- }
-
- // At this point, we know that the extracted files are in the intermediate
- // extracted dir, or in the the original directory.
-
- if (!pathExists(intermediateArchivedDir)) {
- try {
- console.printInfo("Moving " + originalDir + " to " + intermediateArchivedDir);
- fs.rename(originalDir, intermediateArchivedDir);
- } catch (IOException e) {
- throw new HiveException(e);
- }
- } else {
- console.printInfo(intermediateArchivedDir + " already exists. " +
- "Assuming it contains the archived version of the partition");
- }
-
- // If there is a failure from here to until when the metadata is changed,
- // the partition will be empty or throw errors on read.
-
- // If the original location exists here, then it must be the extracted files
- // because in the previous step, we moved the previous original location
- // (containing the archived version of the files) to intermediateArchiveDir
- if (!pathExists(originalDir)) {
- try {
- console.printInfo("Moving " + intermediateExtractedDir + " to " + originalDir);
- fs.rename(intermediateExtractedDir, originalDir);
- } catch (IOException e) {
- throw new HiveException(e);
- }
- } else {
- console.printInfo(originalDir + " already exists. " +
- "Assuming it contains the extracted files in the partition");
- }
-
- for(Partition p: partitions) {
- setUnArchived(p);
- try {
- // TODO: catalog
- db.alterPartition(simpleDesc.getTableName(), p, null, true);
- } catch (InvalidOperationException e) {
- throw new HiveException(e);
- }
- }
-
- // If a failure happens here, the intermediate archive files won't be
- // deleted. The user will need to call unarchive again to clear those up.
- if(pathExists(intermediateArchivedDir)) {
- deleteDir(intermediateArchivedDir, db.getDatabase(tbl.getDbName()));
- }
-
- if(recovery) {
- console.printInfo("Recovery after UNARCHIVE succeeded");
- }
-
- return 0;
- }
-
- private void checkArchiveProperty(int partSpecLevel,
- boolean recovery, Partition p) throws HiveException {
- if (!ArchiveUtils.isArchived(p) && !recovery) {
- throw new HiveException("Partition " + p.getName()
- + " is not archived.");
- }
- int archiveLevel = ArchiveUtils.getArchivingLevel(p);
- if (partSpecLevel > archiveLevel) {
- throw new HiveException("Partition " + p.getName()
- + " is archived at level " + archiveLevel
- + ", and given partspec only has " + partSpecLevel
- + " specs.");
- }
- }
-
- private int compact(Hive db, AlterTableSimpleDesc desc) throws HiveException {
-
- Table tbl = db.getTable(desc.getTableName());
- if (!AcidUtils.isTransactionalTable(tbl)) {
- throw new HiveException(ErrorMsg.NONACID_COMPACTION_NOT_SUPPORTED, tbl.getDbName(),
- tbl.getTableName());
- }
-
- String partName = null;
- if (desc.getPartSpec() == null) {
- // Compaction can only be done on the whole table if the table is non-partitioned.
- if (tbl.isPartitioned()) {
- throw new HiveException(ErrorMsg.NO_COMPACTION_PARTITION);
- }
- } else {
- Map<String, String> partSpec = desc.getPartSpec();
- List<Partition> partitions = db.getPartitions(tbl, partSpec);
- if (partitions.size() > 1) {
- throw new HiveException(ErrorMsg.TOO_MANY_COMPACTION_PARTITIONS);
- } else if (partitions.size() == 0) {
- throw new HiveException(ErrorMsg.INVALID_PARTITION_SPEC);
- }
- partName = partitions.get(0).getName();
- }
- CompactionResponse resp = db.compact2(tbl.getDbName(), tbl.getTableName(), partName,
- desc.getCompactionType(), desc.getProps());
- if(resp.isAccepted()) {
- console.printInfo("Compaction enqueued with id " + resp.getId());
- }
- else {
- console.printInfo("Compaction already enqueued with id " + resp.getId() +
- "; State is " + resp.getState());
- }
- if(desc.isBlocking() && resp.isAccepted()) {
- StringBuilder progressDots = new StringBuilder();
- long waitTimeMs = 1000;
- wait: while (true) {
- //double wait time until 5min
- waitTimeMs = waitTimeMs*2;
- waitTimeMs = waitTimeMs < 5*60*1000 ? waitTimeMs : 5*60*1000;
- try {
- Thread.sleep(waitTimeMs);
- }
- catch(InterruptedException ex) {
- console.printInfo("Interrupted while waiting for compaction with id=" + resp.getId());
- break;
- }
- //this could be expensive when there are a lot of compactions....
- //todo: update to search by ID once HIVE-13353 is done
- ShowCompactResponse allCompactions = db.showCompactions();
- for(ShowCompactResponseElement compaction : allCompactions.getCompacts()) {
- if (resp.getId() != compaction.getId()) {
- continue;
- }
- switch (compaction.getState()) {
- case TxnStore.WORKING_RESPONSE:
- case TxnStore.INITIATED_RESPONSE:
- //still working
- console.printInfo(progressDots.toString());
- progressDots.append(".");
- continue wait;
- default:
- //done
- console.printInfo("Compaction with id " + resp.getId() + " finished with status: " + compaction.getState());
- break wait;
- }
- }
- }
- }
- return 0;
- }
-
- /**
* MetastoreCheck, see if the data in the metastore matches what is on the
* dfs. Current version checks for tables and partitions that are either
* missing on disk on in the metastore.
@@ -1083,130 +217,6 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
}
/**
- * Alter a given table.
- *
- * @param db
- * The database in question.
- * @param alterTbl
- * This is the table we're altering.
- * @return Returns 0 when execution succeeds and above 0 if it fails.
- * @throws HiveException
- * Throws this exception if an unexpected error occurs.
- */
- private int alterTable(Hive db, AlterTableDesc alterTbl) throws HiveException {
- if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.RENAME) {
- String names[] = Utilities.getDbTableName(alterTbl.getOldName());
- if (Utils.isBootstrapDumpInProgress(db, names[0])) {
- LOG.error("DDLTask: Rename Table not allowed as bootstrap dump in progress");
- throw new HiveException("Rename Table: Not allowed as bootstrap dump in progress");
- }
- }
-
- // alter the table
- Table tbl = db.getTable(alterTbl.getOldName());
-
- List<Partition> allPartitions = null;
- if (alterTbl.getPartSpec() != null) {
- Map<String, String> partSpec = alterTbl.getPartSpec();
- if (DDLSemanticAnalyzer.isFullSpec(tbl, partSpec)) {
- allPartitions = new ArrayList<Partition>();
- Partition part = db.getPartition(tbl, partSpec, false);
- if (part == null) {
- // User provided a fully specified partition spec but it doesn't exist, fail.
- throw new HiveException(ErrorMsg.INVALID_PARTITION,
- StringUtils.join(alterTbl.getPartSpec().keySet(), ',') + " for table " + alterTbl.getOldName());
-
- }
- allPartitions.add(part);
- } else {
- // DDLSemanticAnalyzer has already checked if partial partition specs are allowed,
- // thus we should not need to check it here.
- allPartitions = db.getPartitions(tbl, alterTbl.getPartSpec());
- }
- }
-
- // Don't change the table object returned by the metastore, as we'll mess with it's caches.
- Table oldTbl = tbl;
- tbl = oldTbl.copy();
- // Handle child tasks here. We could add them directly whereever we need,
- // but let's make it a little bit more explicit.
- if (allPartitions != null) {
- // Alter all partitions
- for (Partition part : allPartitions) {
- addChildTasks(alterTableOrSinglePartition(alterTbl, tbl, part));
- }
- } else {
- // Just alter the table
- addChildTasks(alterTableOrSinglePartition(alterTbl, tbl, null));
- }
-
- if (allPartitions == null) {
- updateModifiedParameters(tbl.getTTable().getParameters(), conf);
- tbl.checkValidity(conf);
- } else {
- for (Partition tmpPart: allPartitions) {
- updateModifiedParameters(tmpPart.getParameters(), conf);
- }
- }
-
- try {
- EnvironmentContext environmentContext = alterTbl.getEnvironmentContext();
- if (environmentContext == null) {
- environmentContext = new EnvironmentContext();
- }
- environmentContext.putToProperties(HiveMetaHook.ALTER_TABLE_OPERATION_TYPE, alterTbl.getOp().name());
- if (allPartitions == null) {
- long writeId = alterTbl.getWriteId() != null ? alterTbl.getWriteId() : 0;
- if (alterTbl.getReplicationSpec() != null &&
- alterTbl.getReplicationSpec().isMigratingToTxnTable()) {
- Long tmpWriteId = ReplUtils.getMigrationCurrentTblWriteId(conf);
- if (tmpWriteId == null) {
- throw new HiveException("DDLTask : Write id is not set in the config by open txn task for migration");
- }
- writeId = tmpWriteId;
- }
- db.alterTable(alterTbl.getOldName(), tbl, alterTbl.getIsCascade(), environmentContext,
- true, writeId);
- } else {
- // Note: this is necessary for UPDATE_STATISTICS command, that operates via ADDPROPS (why?).
- // For any other updates, we don't want to do txn check on partitions when altering table.
- boolean isTxn = false;
- if (alterTbl.getPartSpec() != null && alterTbl.getOp() == AlterTableTypes.ADDPROPS) {
- // ADDPROPS is used to add replication properties like repl.last.id, which isn't
- // transactional change. In case of replication check for transactional properties
- // explicitly.
- Map<String, String> props = alterTbl.getProps();
- if (alterTbl.getReplicationSpec() != null && alterTbl.getReplicationSpec().isInReplicationScope()) {
- isTxn = (props.get(StatsSetupConst.COLUMN_STATS_ACCURATE) != null);
- } else {
- isTxn = true;
- }
- }
- db.alterPartitions(Warehouse.getQualifiedName(tbl.getTTable()), allPartitions, environmentContext, isTxn);
- }
- } catch (InvalidOperationException e) {
- LOG.error("alter table: ", e);
- throw new HiveException(e, ErrorMsg.GENERIC_ERROR);
- }
-
- // This is kind of hacky - the read entity contains the old table, whereas
- // the write entity
- // contains the new table. This is needed for rename - both the old and the
- // new table names are
- // passed
- // Don't acquire locks for any of these, we have already asked for them in DDLSemanticAnalyzer.
- if (allPartitions != null ) {
- for (Partition tmpPart: allPartitions) {
- work.getInputs().add(new ReadEntity(tmpPart));
- addIfAbsentByName(new WriteEntity(tmpPart, WriteEntity.WriteType.DDL_NO_LOCK));
- }
- } else {
- work.getInputs().add(new ReadEntity(oldTbl));
- addIfAbsentByName(new WriteEntity(tbl, WriteEntity.WriteType.DDL_NO_LOCK));
- }
- return 0;
- }
- /**
* There are many places where "duplicate" Read/WriteEnity objects are added. The way this was
* initially implemented, the duplicate just replaced the previous object.
* (work.getOutputs() is a Set and WriteEntity#equals() relies on name)
@@ -1235,221 +245,6 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
outputs.add(newWriteEntity);
return true;
}
- private boolean addIfAbsentByName(WriteEntity newWriteEntity) {
- return addIfAbsentByName(newWriteEntity, work.getOutputs());
- }
-
- private void addChildTasks(List<Task<?>> extraTasks) {
- if (extraTasks == null) {
- return;
- }
- for (Task<?> newTask : extraTasks) {
- addDependentTask(newTask);
- }
- }
-
- private boolean isSchemaEvolutionEnabled(Table tbl) {
- boolean isAcid = AcidUtils.isTablePropertyTransactional(tbl.getMetadata());
- if (isAcid || HiveConf.getBoolVar(conf, ConfVars.HIVE_SCHEMA_EVOLUTION)) {
- return true;
- }
- return false;
- }
-
-
- private static StorageDescriptor retrieveStorageDescriptor(Table tbl, Partition part) {
- return (part == null ? tbl.getTTable().getSd() : part.getTPartition().getSd());
- }
-
- private List<Task<?>> alterTableOrSinglePartition(AlterTableDesc alterTbl, Table tbl,
- Partition part) throws HiveException {
- EnvironmentContext environmentContext = alterTbl.getEnvironmentContext();
- if (environmentContext == null) {
- environmentContext = new EnvironmentContext();
- alterTbl.setEnvironmentContext(environmentContext);
- }
- // do not need update stats in alter table/partition operations
- if (environmentContext.getProperties() == null ||
- environmentContext.getProperties().get(StatsSetupConst.DO_NOT_UPDATE_STATS) == null) {
- environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE);
- }
-
- if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.RENAME) {
- tbl.setDbName(Utilities.getDatabaseName(alterTbl.getNewName()));
- tbl.setTableName(Utilities.getTableName(alterTbl.getNewName()));
- } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.ADDPROPS) {
- return alterTableAddProps(alterTbl, tbl, part, environmentContext);
- } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.DROPPROPS) {
- return alterTableDropProps(alterTbl, tbl, part, environmentContext);
- } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.OWNER) {
- if (alterTbl.getOwnerPrincipal() != null) {
- tbl.setOwner(alterTbl.getOwnerPrincipal().getName());
- tbl.setOwnerType(alterTbl.getOwnerPrincipal().getType());
- }
- } else {
- throw new HiveException(ErrorMsg.UNSUPPORTED_ALTER_TBL_OP, alterTbl.getOp().toString());
- }
-
- return null;
- }
-
- private List<Task<?>> alterTableDropProps(AlterTableDesc alterTbl, Table tbl,
- Partition part, EnvironmentContext environmentContext) throws HiveException {
- if (StatsSetupConst.USER.equals(environmentContext.getProperties()
- .get(StatsSetupConst.STATS_GENERATED))) {
- // drop a stats parameter, which triggers recompute stats update automatically
- environmentContext.getProperties().remove(StatsSetupConst.DO_NOT_UPDATE_STATS);
- }
-
- List<Task<?>> result = null;
- if (part == null) {
- Set<String> removedSet = alterTbl.getProps().keySet();
- boolean isFromMmTable = AcidUtils.isInsertOnlyTable(tbl.getParameters()),
- isRemoved = AcidUtils.isRemovedInsertOnlyTable(removedSet);
- if (isFromMmTable && isRemoved) {
- throw new HiveException("Cannot convert an ACID table to non-ACID");
- }
-
- // Check if external table property being removed
- if (removedSet.contains("EXTERNAL") && tbl.getTableType() == TableType.EXTERNAL_TABLE) {
- tbl.setTableType(TableType.MANAGED_TABLE);
- }
- }
- Iterator<String> keyItr = alterTbl.getProps().keySet().iterator();
- while (keyItr.hasNext()) {
- if (part != null) {
- part.getTPartition().getParameters().remove(keyItr.next());
- } else {
- tbl.getTTable().getParameters().remove(keyItr.next());
- }
- }
- return result;
- }
-
- private void checkMmLb(Table tbl) throws HiveException {
- if (!tbl.isStoredAsSubDirectories()) {
- return;
- }
- // TODO [MM gap?]: by design; no-one seems to use LB tables. They will work, but not convert.
- // It's possible to work around this by re-creating and re-inserting the table.
- throw new HiveException("Converting list bucketed tables stored as subdirectories "
- + " to MM is not supported. Please re-create a table in the desired format.");
- }
-
- private void checkMmLb(Partition part) throws HiveException {
- if (!part.isStoredAsSubDirectories()) {
- return;
- }
- throw new HiveException("Converting list bucketed tables stored as subdirectories "
- + " to MM is not supported. Please re-create a table in the desired format.");
- }
-
- private List<Task<?>> generateAddMmTasks(Table tbl, Long writeId) throws HiveException {
- // We will move all the files in the table/partition directories into the first MM
- // directory, then commit the first write ID.
- List<Path> srcs = new ArrayList<>(), tgts = new ArrayList<>();
- if (writeId == null) {
- throw new HiveException("Internal error - write ID not set for MM conversion");
- }
-
- int stmtId = 0;
- String mmDir = AcidUtils.deltaSubdir(writeId, writeId, stmtId);
-
- Hive db = getHive();
- if (tbl.getPartitionKeys().size() > 0) {
- PartitionIterable parts = new PartitionIterable(db, tbl, null,
- HiveConf.getIntVar(conf, ConfVars.METASTORE_BATCH_RETRIEVE_MAX));
- Iterator<Partition> partIter = parts.iterator();
- while (partIter.hasNext()) {
- Partition part = partIter.next();
- checkMmLb(part);
- Path src = part.getDataLocation(), tgt = new Path(src, mmDir);
- srcs.add(src);
- tgts.add(tgt);
- if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
- Utilities.FILE_OP_LOGGER.trace("Will move " + src + " to " + tgt);
- }
- }
- } else {
- checkMmLb(tbl);
- Path src = tbl.getDataLocation(), tgt = new Path(src, mmDir);
- srcs.add(src);
- tgts.add(tgt);
- if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
- Utilities.FILE_OP_LOGGER.trace("Will move " + src + " to " + tgt);
- }
- }
- // Don't set inputs and outputs - the locks have already been taken so it's pointless.
- MoveWork mw = new MoveWork(null, null, null, null, false);
- mw.setMultiFilesDesc(new LoadMultiFilesDesc(srcs, tgts, true, null, null));
- return Lists.<Task<?>>newArrayList(TaskFactory.get(mw));
- }
-
- private List<Task<?>> alterTableAddProps(AlterTableDesc alterTbl, Table tbl,
- Partition part, EnvironmentContext environmentContext) throws HiveException {
- if (StatsSetupConst.USER.equals(environmentContext.getProperties()
- .get(StatsSetupConst.STATS_GENERATED))) {
- environmentContext.getProperties().remove(StatsSetupConst.DO_NOT_UPDATE_STATS);
- }
- List<Task<?>> result = null;
- if (part != null) {
- part.getTPartition().getParameters().putAll(alterTbl.getProps());
- } else {
- boolean isFromMmTable = AcidUtils.isInsertOnlyTable(tbl.getParameters());
- Boolean isToMmTable = AcidUtils.isToInsertOnlyTable(tbl, alterTbl.getProps());
- if (isToMmTable != null) {
- if (!isFromMmTable && isToMmTable) {
- if (!HiveConf.getBoolVar(conf, ConfVars.HIVE_MM_ALLOW_ORIGINALS)) {
- result = generateAddMmTasks(tbl, alterTbl.getWriteId());
- } else {
- if (tbl.getPartitionKeys().size() > 0) {
- Hive db = getHive();
- PartitionIterable parts = new PartitionIterable(db, tbl, null,
- HiveConf.getIntVar(conf, ConfVars.METASTORE_BATCH_RETRIEVE_MAX));
- Iterator<Partition> partIter = parts.iterator();
- while (partIter.hasNext()) {
- Partition part0 = partIter.next();
- checkMmLb(part0);
- }
- } else {
- checkMmLb(tbl);
- }
- }
- } else if (isFromMmTable && !isToMmTable) {
- throw new HiveException("Cannot convert an ACID table to non-ACID");
- }
- }
-
- // Converting to/from external table
- String externalProp = alterTbl.getProps().get("EXTERNAL");
- if (externalProp != null) {
- if (Boolean.parseBoolean(externalProp) && tbl.getTableType() == TableType.MANAGED_TABLE) {
- tbl.setTableType(TableType.EXTERNAL_TABLE);
- } else if (!Boolean.parseBoolean(externalProp) && tbl.getTableType() == TableType.EXTERNAL_TABLE) {
- tbl.setTableType(TableType.MANAGED_TABLE);
- }
- }
-
- tbl.getTTable().getParameters().putAll(alterTbl.getProps());
- }
- return result;
- }
-
- /**
- * Update last_modified_by and last_modified_time parameters in parameter map.
- *
- * @param params
- * Parameters.
- * @param conf
- * HiveConf of session
- */
- private boolean updateModifiedParameters(Map<String, String> params, HiveConf conf) throws HiveException {
- String user = null;
- user = SessionState.getUserFromAuthenticator();
- params.put("last_modified_by", user);
- params.put("last_modified_time", Long.toString(System.currentTimeMillis() / 1000));
- return true;
- }
/**
* Check if the given serde is valid.
@@ -1477,45 +272,6 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
return "DDL";
}
- /**
- * Validate if the given table/partition is eligible for update
- *
- * @param db Database.
- * @param tableName Table name of format db.table
- * @param partSpec Partition spec for the partition
- * @param replicationSpec Replications specification
- *
- * @return boolean true if allow the operation
- * @throws HiveException
- */
- private boolean allowOperationInReplicationScope(Hive db, String tableName,
- Map<String, String> partSpec, ReplicationSpec replicationSpec) throws HiveException {
- if ((null == replicationSpec) || (!replicationSpec.isInReplicationScope())) {
- // Always allow the operation if it is not in replication scope.
- return true;
- }
- // If the table/partition exist and is older than the event, then just apply
- // the event else noop.
- Table existingTable = db.getTable(tableName, false);
- if ((existingTable != null)
- && replicationSpec.allowEventReplacementInto(existingTable.getParameters())) {
- // Table exists and is older than the update. Now, need to ensure if update allowed on the
- // partition.
- if (partSpec != null) {
- Partition existingPtn = db.getPartition(existingTable, partSpec, false);
- return ((existingPtn != null)
- && replicationSpec.allowEventReplacementInto(existingPtn.getParameters()));
- }
-
- // Replacement is allowed as the existing table is older than event
- return true;
- }
-
- // The table is missing either due to drop/rename which follows the operation.
- // Or the existing table is newer than our update. So, don't allow the update.
- return false;
- }
-
private int remFirstIncPendFlag(Hive hive, ReplRemoveFirstIncLoadPendFlagDesc desc) throws HiveException, TException {
String dbNameOrPattern = desc.getDatabaseName();
Map<String, String> parameters;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
index 078691c..ae9d040 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java
@@ -44,7 +44,7 @@ public class ExportTask extends Task<ExportWork> implements Serializable {
}
@Override
- protected int execute(DriverContext driverContext) {
+ public int execute(DriverContext driverContext) {
try {
// Also creates the root directory
TableExport.Paths exportPaths = new TableExport.Paths(
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
index dcf569f..6b3635d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java
@@ -133,7 +133,7 @@ public class ReplCopyTask extends Task<ReplCopyWork> implements Serializable {
}
@Override
- protected int execute(DriverContext driverContext) {
+ public int execute(DriverContext driverContext) {
LOG.debug("ReplCopyTask.execute()");
FileSystem dstFs = null;
Path toPath = null;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
index 956c4ff..f542105 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java
@@ -225,7 +225,7 @@ public abstract class Task<T extends Serializable> implements Serializable, Node
*
* @return status of executing the task
*/
- protected abstract int execute(DriverContext driverContext);
+ public abstract int execute(DriverContext driverContext);
public boolean isRootTask() {
return rootTask;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ExternalTableCopyTaskBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ExternalTableCopyTaskBuilder.java
index 6bc3cd0..0ee7425 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ExternalTableCopyTaskBuilder.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ExternalTableCopyTaskBuilder.java
@@ -163,7 +163,7 @@ public class ExternalTableCopyTaskBuilder {
}
@Override
- protected int execute(DriverContext driverContext) {
+ public int execute(DriverContext driverContext) {
String distCpDoAsUser = conf.getVar(HiveConf.ConfVars.HIVE_DISTCP_DOAS_USER);
Path sourcePath = work.fullyQualifiedSourcePath;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
index 8c963be..453929a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
@@ -108,7 +108,7 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable {
}
@Override
- protected int execute(DriverContext driverContext) {
+ public int execute(DriverContext driverContext) {
try {
Hive hiveDb = getHive();
Path dumpRoot = new Path(conf.getVar(HiveConf.ConfVars.REPLDIR), getNextDumpDir());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java
index f4f98a6..7d2c7c9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java
@@ -91,7 +91,7 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
}
@Override
- protected int execute(DriverContext driverContext) {
+ public int execute(DriverContext driverContext) {
Task<? extends Serializable> rootTask = work.getRootTask();
if (rootTask != null) {
rootTask.setChildTasks(null);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/incremental/IncrementalLoadTasksBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/incremental/IncrementalLoadTasksBuilder.java
index 43def23..f6975d3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/incremental/IncrementalLoadTasksBuilder.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/incremental/IncrementalLoadTasksBuilder.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.metastore.api.ReplLastIdInfo;
import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.ddl.DDLWork2;
import org.apache.hadoop.hive.ql.ddl.database.AlterDatabaseDesc;
+import org.apache.hadoop.hive.ql.ddl.table.misc.AlterTableSetPropertiesDesc;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.exec.repl.ReplStateLogWork;
@@ -46,7 +47,6 @@ import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
import org.apache.hadoop.hive.ql.parse.repl.load.UpdatedMetaDataTracker;
import org.apache.hadoop.hive.ql.parse.repl.load.log.IncrementalLoadLogger;
import org.apache.hadoop.hive.ql.parse.repl.load.message.MessageHandler;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc;
import org.apache.hadoop.hive.ql.plan.DDLWork;
import org.apache.hadoop.hive.ql.plan.DependencyCollectionWork;
import org.apache.hadoop.hive.ql.plan.ReplTxnWork;
@@ -259,13 +259,11 @@ public class IncrementalLoadTasksBuilder {
HashMap<String, String> mapProp = new HashMap<>();
mapProp.put(ReplicationSpec.KEY.CURR_STATE_ID.toString(), replState);
- AlterTableDesc alterTblDesc = new AlterTableDesc(
- AlterTableDesc.AlterTableTypes.ADDPROPS, new ReplicationSpec(replState, replState));
- alterTblDesc.setProps(mapProp);
- alterTblDesc.setOldName(StatsUtils.getFullyQualifiedTableName(dbName, tableName));
- alterTblDesc.setPartSpec((HashMap<String, String>) partSpec);
+ String fqTableName = StatsUtils.getFullyQualifiedTableName(dbName, tableName);
+ AlterTableSetPropertiesDesc alterTblDesc = new AlterTableSetPropertiesDesc(fqTableName, partSpec,
+ new ReplicationSpec(replState, replState), false, mapProp, false, false, null);
- Task<? extends Serializable> updateReplIdTask = TaskFactory.get(new DDLWork(inputs, outputs, alterTblDesc), conf);
+ Task<? extends Serializable> updateReplIdTask = TaskFactory.get(new DDLWork2(inputs, outputs, alterTblDesc), conf);
// Link the update repl state task with dependency collection task
if (preCursor != null) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/ReplUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/ReplUtils.java
index 2fcbe64..27127e0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/ReplUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/ReplUtils.java
@@ -26,6 +26,8 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.ddl.DDLWork2;
+import org.apache.hadoop.hive.ql.ddl.table.misc.AlterTableSetPropertiesDesc;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.exec.repl.ReplStateLogWork;
@@ -34,9 +36,7 @@ import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.parse.repl.ReplLogger;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc;
import org.apache.hadoop.hive.ql.plan.ColumnStatsUpdateWork;
-import org.apache.hadoop.hive.ql.plan.DDLWork;
import org.apache.hadoop.hive.ql.plan.ReplTxnWork;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
@@ -59,7 +59,6 @@ import java.io.Serializable;
import static org.apache.hadoop.hive.ql.util.HiveStrictManagedMigration.TableMigrationOption.MANAGED;
-
public class ReplUtils {
public static final String LAST_REPL_ID_KEY = "hive.repl.last.repl.id";
@@ -144,14 +143,10 @@ public class ReplUtils {
HashMap<String, String> mapProp = new HashMap<>();
mapProp.put(REPL_CHECKPOINT_KEY, dumpRoot);
- AlterTableDesc alterTblDesc = new AlterTableDesc(AlterTableDesc.AlterTableTypes.ADDPROPS);
- alterTblDesc.setProps(mapProp);
- alterTblDesc.setOldName(
- StatsUtils.getFullyQualifiedTableName(tableDesc.getDatabaseName(), tableDesc.getTableName()));
- if (partSpec != null) {
- alterTblDesc.setPartSpec(partSpec);
- }
- return TaskFactory.get(new DDLWork(new HashSet<>(), new HashSet<>(), alterTblDesc), conf);
+ String fqTableName = StatsUtils.getFullyQualifiedTableName(tableDesc.getDatabaseName(), tableDesc.getTableName());
+ AlterTableSetPropertiesDesc alterTblDesc = new AlterTableSetPropertiesDesc(fqTableName, partSpec, null, false,
+ mapProp, false, false, null);
+ return TaskFactory.get(new DDLWork2(new HashSet<>(), new HashSet<>(), alterTblDesc), conf);
}
public static boolean replCkptStatus(String dbName, Map<String, String> props, String dumpRoot)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
index 7f8f9a7..f75eb85 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java
@@ -20,10 +20,10 @@ package org.apache.hadoop.hive.ql.hooks;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
import org.apache.hadoop.hive.ql.metadata.DummyPartition;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc;
import java.io.Serializable;
@@ -195,7 +195,7 @@ public class WriteEntity extends Entity implements Serializable {
* @param op Operation type from the alter table description
* @return the write type this should use.
*/
- public static WriteType determineAlterTableWriteType(AlterTableDesc.AlterTableTypes op) {
+ public static WriteType determineAlterTableWriteType(AlterTableType op) {
switch (op) {
case RENAME_COLUMN:
case CLUSTERED_BY:
@@ -207,14 +207,14 @@ public class WriteEntity extends Entity implements Serializable {
case REPLACE_COLUMNS:
case ARCHIVE:
case UNARCHIVE:
- case SET_LOCATION:
+ case ALTERLOCATION:
case DROPPARTITION:
case RENAMEPARTITION:
case SKEWED_BY:
case SET_SKEWED_LOCATION:
case INTO_BUCKETS:
case ALTERPARTITION:
- case ADD_COLUMNS:
+ case ADDCOLS:
case RENAME:
case TRUNCATE:
case MERGEFILES:
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java
index 3482cfc..9d5ba3d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java
@@ -30,6 +30,7 @@ import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
@@ -37,6 +38,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.hive.shims.ShimLoader;
@@ -123,4 +125,13 @@ public class HdfsUtils {
// Note - this makes assumptions that are DFS-specific; DFS::getDefaultPort is not visible.
return (defaultPort == -1) ? (port == NameNode.DEFAULT_PORT) : (port == defaultPort);
}
+
+ public static boolean pathExists(Path p, Configuration conf) throws HiveException {
+ try {
+ FileSystem fs = p.getFileSystem(conf);
+ return fs.exists(p);
+ } catch (IOException e) {
+ throw new HiveException(e);
+ }
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java
index 76415cf..cd7b69d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hive.ql.ddl.DDLTask2;
import org.apache.hadoop.hive.ql.ddl.DDLWork2;
import org.apache.hadoop.hive.ql.ddl.table.creation.CreateTableLikeDesc;
import org.apache.hadoop.hive.ql.ddl.table.creation.DropTableDesc;
+import org.apache.hadoop.hive.ql.ddl.table.misc.AlterTableSetPropertiesDesc;
import org.apache.hadoop.hive.ql.exec.StatsTask;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
@@ -49,8 +50,6 @@ import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.HiveUtils;
import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc;
-import org.apache.hadoop.hive.ql.plan.DDLWork;
import org.apache.hadoop.hive.ql.plan.ExportWork;
import org.apache.hadoop.hive.ql.session.SessionState;
@@ -189,12 +188,11 @@ public class AcidExportSemanticAnalyzer extends RewriteSemanticAnalyzer {
// do it after populating temp table so that it's written as non-transactional table but
// update props before export so that export archive metadata has these props. This way when
// IMPORT is done for this archive and target table doesn't exist, it will be created as Acid.
- AlterTableDesc alterTblDesc = new AlterTableDesc(AlterTableDesc.AlterTableTypes.ADDPROPS);
Map<String, String> mapProps = new HashMap<>();
mapProps.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, Boolean.TRUE.toString());
- alterTblDesc.setProps(mapProps);
- alterTblDesc.setOldName(newTableName);
- addExportTask(rootTasks, exportTask, TaskFactory.get(new DDLWork(getInputs(), getOutputs(), alterTblDesc)));
+ AlterTableSetPropertiesDesc alterTblDesc = new AlterTableSetPropertiesDesc(newTableName, null, null, false,
+ mapProps, false, false, null);
+ addExportTask(rootTasks, exportTask, TaskFactory.get(new DDLWork2(getInputs(), getOutputs(), alterTblDesc)));
// Now make a task to drop temp table
// {@link DDLSemanticAnalyzer#analyzeDropTable(ASTNode ast, TableType expectedType)
@@ -254,7 +252,7 @@ public class AcidExportSemanticAnalyzer extends RewriteSemanticAnalyzer {
* Makes the exportTask run after all other tasks of the "insert into T ..." are done.
*/
private void addExportTask(List<Task<?>> rootTasks,
- Task<ExportWork> exportTask, Task<DDLWork> alterTable) {
+ Task<ExportWork> exportTask, Task<DDLWork2> alterTable) {
for (Task<? extends Serializable> t : rootTasks) {
if (t.getNumChild() <= 0) {
//todo: ConditionalTask#addDependentTask(Task) doesn't do the right thing: HIVE-18978
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index a58ac2f..f13593e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@ -71,6 +71,7 @@ import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.ddl.DDLDesc;
import org.apache.hadoop.hive.ql.ddl.DDLWork2;
import org.apache.hadoop.hive.ql.ddl.database.AlterDatabaseDesc;
import org.apache.hadoop.hive.ql.ddl.database.CreateDatabaseDesc;
@@ -93,6 +94,7 @@ import org.apache.hadoop.hive.ql.ddl.process.KillQueriesDesc;
import org.apache.hadoop.hive.ql.ddl.process.ShowCompactionsDesc;
import org.apache.hadoop.hive.ql.ddl.process.ShowTransactionsDesc;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
import org.apache.hadoop.hive.ql.ddl.table.column.AlterTableAddColumnsDesc;
import org.apache.hadoop.hive.ql.ddl.table.column.AlterTableChangeColumnDesc;
import org.apache.hadoop.hive.ql.ddl.table.column.AlterTableReplaceColumnsDesc;
@@ -110,6 +112,11 @@ import org.apache.hadoop.hive.ql.ddl.table.info.ShowTablesDesc;
import org.apache.hadoop.hive.ql.ddl.table.lock.LockTableDesc;
import org.apache.hadoop.hive.ql.ddl.table.lock.ShowLocksDesc;
import org.apache.hadoop.hive.ql.ddl.table.lock.UnlockTableDesc;
+import org.apache.hadoop.hive.ql.ddl.table.misc.AlterTableRenameDesc;
+import org.apache.hadoop.hive.ql.ddl.table.misc.AlterTableSetOwnerDesc;
+import org.apache.hadoop.hive.ql.ddl.table.misc.AlterTableSetPropertiesDesc;
+import org.apache.hadoop.hive.ql.ddl.table.misc.AlterTableTouchDesc;
+import org.apache.hadoop.hive.ql.ddl.table.misc.AlterTableUnsetPropertiesDesc;
import org.apache.hadoop.hive.ql.ddl.table.misc.TruncateTableDesc;
import org.apache.hadoop.hive.ql.ddl.table.partition.AlterTableAddPartitionDesc;
import org.apache.hadoop.hive.ql.ddl.table.partition.AlterTableAlterPartitionDesc;
@@ -117,17 +124,21 @@ import org.apache.hadoop.hive.ql.ddl.table.partition.AlterTableDropPartitionDesc
import org.apache.hadoop.hive.ql.ddl.table.partition.AlterTableExchangePartitionsDesc;
import org.apache.hadoop.hive.ql.ddl.table.partition.AlterTableRenamePartitionDesc;
import org.apache.hadoop.hive.ql.ddl.table.partition.ShowPartitionsDesc;
+import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableArchiveDesc;
import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableClusteredByDesc;
+import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableCompactDesc;
import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableIntoBucketsDesc;
import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableNotClusteredDesc;
import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableNotSkewedDesc;
import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableNotSortedDesc;
+import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableConcatenateDesc;
import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableSetFileFormatDesc;
import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableSetLocationDesc;
import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableSetSerdeDesc;
import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableSetSerdePropsDesc;
import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableSetSkewedLocationDesc;
import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableSkewedByDesc;
+import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableUnarchiveDesc;
import org.apache.hadoop.hive.ql.ddl.table.partition.AlterTableAddPartitionDesc.PartitionDesc;
import org.apache.hadoop.hive.ql.ddl.view.AlterMaterializedViewRewriteDesc;
import org.apache.hadoop.hive.ql.ddl.workloadmanagement.AlterPoolAddTriggerDesc;
@@ -173,13 +184,9 @@ import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.authorization.AuthorizationParseUtils;
import org.apache.hadoop.hive.ql.parse.authorization.HiveAuthorizationTaskFactory;
import org.apache.hadoop.hive.ql.parse.authorization.HiveAuthorizationTaskFactoryImpl;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
-import org.apache.hadoop.hive.ql.plan.AlterTableSimpleDesc;
import org.apache.hadoop.hive.ql.plan.BasicStatsWork;
import org.apache.hadoop.hive.ql.plan.CacheMetadataDesc;
import org.apache.hadoop.hive.ql.plan.ColumnStatsUpdateWork;
-import org.apache.hadoop.hive.ql.plan.DDLDesc;
import org.apache.hadoop.hive.ql.plan.DDLDesc.DDLDescWithWriteId;
import org.apache.hadoop.hive.ql.plan.DDLWork;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
@@ -228,7 +235,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
private final HiveAuthorizationTaskFactory hiveAuthorizationTaskFactory;
private WriteEntity alterTableOutput;
// Equivalent to acidSinks, but for DDL operations that change data.
- private DDLDesc.DDLDescWithWriteId ddlDescWithWriteId;
+ private DDLDescWithWriteId ddlDescWithWriteId;
static {
TokenToTypeName.put(HiveParser.TOK_BOOLEAN, serdeConstants.BOOLEAN_TYPE_NAME);
@@ -683,7 +690,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
ColumnStatsUpdateTask cStatsUpdateTask = (ColumnStatsUpdateTask) TaskFactory
.get(columnStatsUpdateWork);
// TODO: doesn't look like this path is actually ever exercised. Maybe this needs to be removed.
- addInputsOutputsAlterTable(tblName, partSpec, AlterTableTypes.UPDATESTATS);
+ addInputsOutputsAlterTable(tblName, partSpec, null, AlterTableType.UPDATESTATS, false);
if (AcidUtils.isTransactionalTable(tbl)) {
setAcidDdlDesc(columnStatsUpdateWork);
}
@@ -1621,7 +1628,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
truncateTblDesc.setInputDir(oldTblPartLoc);
truncateTblDesc.setLbCtx(lbCtx);
- addInputsOutputsAlterTable(tableName, partSpec, AlterTableTypes.TRUNCATE);
+ addInputsOutputsAlterTable(tableName, partSpec, null, AlterTableType.TRUNCATE, false);
ddlWork.setNeedLock(true);
TableDesc tblDesc = Utilities.getTableDesc(table);
// Write the output to temporary directory and move it to the final location at the end
@@ -1669,11 +1676,11 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
return true;
}
- private void validateAlterTableType(Table tbl, AlterTableTypes op) throws SemanticException {
+ private void validateAlterTableType(Table tbl, AlterTableType op) throws SemanticException {
validateAlterTableType(tbl, op, false);
}
- private void validateAlterTableType(Table tbl, AlterTableTypes op, boolean expectView)
+ private void validateAlterTableType(Table tbl, AlterTableType op, boolean expectView)
throws SemanticException {
if (tbl.isView()) {
if (!expectView) {
@@ -1697,7 +1704,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
throw new SemanticException(ErrorMsg.ALTER_COMMAND_FOR_TABLES.getMsg());
}
}
- if (tbl.isNonNative() && !AlterTableTypes.nonNativeTableAllowedTypes.contains(op)) {
+ if (tbl.isNonNative() && !AlterTableType.NON_NATIVE_TABLE_ALLOWED.contains(op)) {
throw new SemanticException(ErrorMsg.ALTER_TABLE_NON_NATIVE.getMsg(tbl.getTableName()));
}
}
@@ -1776,43 +1783,45 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
environmentContext.putToProperties(StatsSetupConst.STATS_GENERATED, StatsSetupConst.USER);
}
}
- AlterTableDesc alterTblDesc = null;
- if (isUnset == true) {
- alterTblDesc = new AlterTableDesc(AlterTableTypes.DROPPROPS, partSpec, expectView);
- if (ast.getChild(1) != null) {
- alterTblDesc.setDropIfExists(true);
+ boolean isToTxn = AcidUtils.isTablePropertyTransactional(mapProp)
+ || mapProp.containsKey(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
+ boolean isExplicitStatsUpdate = changeStatsSucceeded && AcidUtils.isTransactionalTable(getTable(qualified, true));
+ AbstractAlterTableDesc alterTblDesc = null;
+ DDLWork2 ddlWork = null;
+
+ if (isUnset) {
+ boolean dropIfExists = ast.getChild(1) != null;
+ // validate Unset Non Existed Table Properties
+ if (!dropIfExists) {
+ Table tab = getTable(tableName, true);
+ Map<String, String> tableParams = tab.getTTable().getParameters();
+ for (String currKey : mapProp.keySet()) {
+ if (!tableParams.containsKey(currKey)) {
+ String errorMsg = "The following property " + currKey + " does not exist in " + tab.getTableName();
+ throw new SemanticException(
+ ErrorMsg.ALTER_TBL_UNSET_NON_EXIST_PROPERTY.getMsg(errorMsg));
+ }
+ }
}
+
+ alterTblDesc = new AlterTableUnsetPropertiesDesc(tableName, partSpec, null, expectView, mapProp,
+ isExplicitStatsUpdate, environmentContext);
+ addInputsOutputsAlterTable(tableName, partSpec, alterTblDesc, alterTblDesc.getType(), isToTxn);
+ ddlWork = new DDLWork2(getInputs(), getOutputs(), alterTblDesc);
} else {
addPropertyReadEntry(mapProp, inputs);
- alterTblDesc = new AlterTableDesc(AlterTableTypes.ADDPROPS, partSpec, expectView);
+ boolean isAcidConversion = isToTxn && AcidUtils.isFullAcidTable(mapProp)
+ && !AcidUtils.isFullAcidTable(getTable(qualified, true));
+ alterTblDesc = new AlterTableSetPropertiesDesc(tableName, partSpec, null, expectView, mapProp,
+ isExplicitStatsUpdate, isAcidConversion, environmentContext);
+ addInputsOutputsAlterTable(tableName, partSpec, alterTblDesc, alterTblDesc.getType(), isToTxn);
+ ddlWork = new DDLWork2(getInputs(), getOutputs(), alterTblDesc);
}
- alterTblDesc.setProps(mapProp);
- alterTblDesc.setEnvironmentContext(environmentContext);
- alterTblDesc.setOldName(tableName);
-
-
-
- boolean isToTxn = AcidUtils.isTablePropertyTransactional(mapProp)
- || mapProp.containsKey(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
- addInputsOutputsAlterTable(tableName, partSpec, alterTblDesc, isToTxn);
- // This special handling is because we cannot generate write ID for full ACID conversion,
- // it will break the weird 10000001-write-ID logic that is currently in use. However, we do
- // want to generate a write ID for prop changes for existing txn tables, or MM conversion.
- boolean isAcidConversion = isToTxn && AcidUtils.isFullAcidTable(mapProp)
- && !AcidUtils.isFullAcidTable(getTable(qualified, true));
-
- DDLWork ddlWork = new DDLWork(getInputs(), getOutputs(), alterTblDesc);
if (isToTxn) {
- alterTblDesc.setIsFullAcidConversion(isAcidConversion);
- setAcidDdlDesc(alterTblDesc);
ddlWork.setNeedLock(true); // Hmm... why don't many other operations here need locks?
}
- if (changeStatsSucceeded) {
- Table table = getTable(qualified, true);
- if (AcidUtils.isTransactionalTable(table)) {
- alterTblDesc.setIsExplicitStatsUpdate(true);
- setAcidDdlDesc(alterTblDesc);
- }
+ if (isToTxn || isExplicitStatsUpdate) {
+ setAcidDdlDesc(alterTblDesc);
}
rootTasks.add(TaskFactory.get(ddlWork));
@@ -1835,7 +1844,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
Map<String, String> mapProp = getProps((ASTNode) (ast.getChild(0)).getChild(0));
AlterTableSetSerdePropsDesc alterTblDesc = new AlterTableSetSerdePropsDesc(tableName, partSpec, mapProp);
- addInputsOutputsAlterTable(tableName, partSpec, alterTblDesc, AlterTableTypes.SET_SERDE_PROPS, false);
+ addInputsOutputsAlterTable(tableName, partSpec, alterTblDesc, AlterTableType.SET_SERDE_PROPS, false);
rootTasks.add(TaskFactory.get(new DDLWork2(getInputs(), getOutputs(), alterTblDesc)));
}
@@ -1845,7 +1854,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
Map<String, String> props = (ast.getChildCount() > 1) ? getProps((ASTNode) (ast.getChild(1)).getChild(0)) : null;
AlterTableSetSerdeDesc alterTblDesc = new AlterTableSetSerdeDesc(tableName, partSpec, props, serdeName);
- addInputsOutputsAlterTable(tableName, partSpec, alterTblDesc, AlterTableTypes.SET_SERDE, false);
+ addInputsOutputsAlterTable(tableName, partSpec, alterTblDesc, AlterTableType.SET_SERDE, false);
rootTasks.add(TaskFactory.get(new DDLWork2(getInputs(), getOutputs(), alterTblDesc)));
}
@@ -1860,25 +1869,12 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
AlterTableSetFileFormatDesc alterTblDesc = new AlterTableSetFileFormatDesc(tableName, partSpec,
format.getInputFormat(), format.getOutputFormat(), format.getSerde());
- addInputsOutputsAlterTable(tableName, partSpec, alterTblDesc, AlterTableTypes.SET_FILE_FORMAT, false);
+ addInputsOutputsAlterTable(tableName, partSpec, alterTblDesc, AlterTableType.SET_FILE_FORMAT, false);
rootTasks.add(TaskFactory.get(new DDLWork2(getInputs(), getOutputs(), alterTblDesc)));
}
- private WriteType determineAlterTableWriteType(Table tab, AlterTableDesc desc, AlterTableTypes op) {
- boolean convertingToAcid = false;
- if(desc != null && desc.getProps() != null && Boolean.parseBoolean(desc.getProps().get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL))) {
- convertingToAcid = true;
- }
- if(!AcidUtils.isTransactionalTable(tab) && convertingToAcid) {
- //non-acid to transactional conversion (property itself) must be mutexed to prevent concurrent writes.
- // See HIVE-16688 for use cases.
- return WriteType.DDL_EXCLUSIVE;
- }
- return WriteEntity.determineAlterTableWriteType(op);
- }
-
// For the time while all the alter table operations are getting migrated there is a duplication of this method here
- private WriteType determineAlterTableWriteType(Table tab, AbstractAlterTableDesc desc, AlterTableTypes op) {
+ private WriteType determineAlterTableWriteType(Table tab, AbstractAlterTableDesc desc, AlterTableType op) {
boolean convertingToAcid = false;
if (desc != null && desc.getProps() != null &&
Boolean.parseBoolean(desc.getProps().get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL))) {
@@ -1891,101 +1887,9 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
}
return WriteEntity.determineAlterTableWriteType(op);
}
- private void addInputsOutputsAlterTable(String tableName, Map<String, String> partSpec,
- AlterTableTypes op) throws SemanticException {
- addInputsOutputsAlterTable(tableName, partSpec, (AlterTableDesc)null, op, false);
- }
-
- private void addInputsOutputsAlterTable(String tableName, Map<String, String> partSpec,
- AlterTableDesc desc, boolean doForceExclusive) throws SemanticException {
- addInputsOutputsAlterTable(tableName, partSpec, desc, desc.getOp(), doForceExclusive);
- }
-
- private void addInputsOutputsAlterTable(String tableName, Map<String, String> partSpec,
- AlterTableDesc desc) throws SemanticException {
- addInputsOutputsAlterTable(tableName, partSpec, desc, desc.getOp(), false);
- }
-
- private void addInputsOutputsAlterTable(String tableName, Map<String, String> partSpec,
- AlterTableDesc desc, AlterTableTypes op, boolean doForceExclusive) throws SemanticException {
- boolean isCascade = desc != null && desc.getIsCascade();
- boolean alterPartitions = partSpec != null && !partSpec.isEmpty();
- //cascade only occurs at table level then cascade to partition level
- if (isCascade && alterPartitions) {
- throw new SemanticException(
- ErrorMsg.ALTER_TABLE_PARTITION_CASCADE_NOT_SUPPORTED, op.getName());
- }
-
- Table tab = getTable(tableName, true);
- // cascade only occurs with partitioned table
- if (isCascade && !tab.isPartitioned()) {
- throw new SemanticException(
- ErrorMsg.ALTER_TABLE_NON_PARTITIONED_TABLE_CASCADE_NOT_SUPPORTED);
- }
-
- // Determine the lock type to acquire
- WriteEntity.WriteType writeType = doForceExclusive
- ? WriteType.DDL_EXCLUSIVE : determineAlterTableWriteType(tab, desc, op);
-
- if (!alterPartitions) {
- inputs.add(new ReadEntity(tab));
- alterTableOutput = new WriteEntity(tab, writeType);
- outputs.add(alterTableOutput);
- //do not need the lock for partitions since they are covered by the table lock
- if (isCascade) {
- for (Partition part : getPartitions(tab, partSpec, false)) {
- outputs.add(new WriteEntity(part, WriteEntity.WriteType.DDL_NO_LOCK));
- }
- }
- } else {
- ReadEntity re = new ReadEntity(tab);
- // In the case of altering a table for its partitions we don't need to lock the table
- // itself, just the partitions. But the table will have a ReadEntity. So mark that
- // ReadEntity as no lock.
- re.noLockNeeded();
- inputs.add(re);
-
- if (isFullSpec(tab, partSpec)) {
- // Fully specified partition spec
- Partition part = getPartition(tab, partSpec, true);
- outputs.add(new WriteEntity(part, writeType));
- } else {
- // Partial partition spec supplied. Make sure this is allowed.
- if (!AlterTableDesc.doesAlterTableTypeSupportPartialPartitionSpec(op)) {
- throw new SemanticException(
- ErrorMsg.ALTER_TABLE_TYPE_PARTIAL_PARTITION_SPEC_NO_SUPPORTED, op.getName());
- } else if (!conf.getBoolVar(HiveConf.ConfVars.DYNAMICPARTITIONING)) {
- throw new SemanticException(ErrorMsg.DYNAMIC_PARTITION_DISABLED);
- }
-
- for (Partition part : getPartitions(tab, partSpec, true)) {
- outputs.add(new WriteEntity(part, writeType));
- }
- }
- }
-
- if (desc != null) {
- validateAlterTableType(tab, op, desc.getExpectView());
-
- // validate Unset Non Existed Table Properties
- if (op == AlterTableDesc.AlterTableTypes.DROPPROPS && !desc.getIsDropIfExists()) {
- Map<String, String> tableParams = tab.getTTable().getParameters();
- for (String currKey : desc.getProps().keySet()) {
- if (!tableParams.containsKey(currKey)) {
- String errorMsg =
- "The following property " + currKey +
- " does not exist in " + tab.getTableName();
- throw new SemanticException(
- ErrorMsg.ALTER_TBL_UNSET_NON_EXIST_PROPERTY.getMsg(errorMsg));
- }
- }
- }
- }
- }
- // For the time while all the alter table operations are getting migrated there is a duplication of this method here
private void addInputsOutputsAlterTable(String tableName, Map<String, String> partSpec,
- AbstractAlterTableDesc desc, AlterTableTypes op, boolean doForceExclusive) throws SemanticException {
+ AbstractAlterTableDesc desc, AlterTableType op, boolean doForceExclusive) throws SemanticException {
boolean isCascade = desc != null && desc.isCascade();
boolean alterPartitions = partSpec != null && !partSpec.isEmpty();
//cascade only occurs at table level then cascade to partition level
@@ -2029,7 +1933,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
outputs.add(new WriteEntity(part, writeType));
} else {
// Partial partition spec supplied. Make sure this is allowed.
- if (!AlterTableDesc.doesAlterTableTypeSupportPartialPartitionSpec(op)) {
+ if (!AlterTableType.SUPPORT_PARTIAL_PARTITION_SPEC.contains(op)) {
throw new SemanticException(
ErrorMsg.ALTER_TABLE_TYPE_PARTIAL_PARTITION_SPEC_NO_SUPPORTED, op.getName());
} else if (!conf.getBoolVar(HiveConf.ConfVars.DYNAMICPARTITIONING)) {
@@ -2044,20 +1948,6 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
if (desc != null) {
validateAlterTableType(tab, op, desc.expectView());
-
- // validate Unset Non Existed Table Properties
-/* if (op == AlterTableDesc.AlterTableTypes.DROPPROPS && !desc.getIsDropIfExists()) {
- Map<String, String> tableParams = tab.getTTable().getParameters();
- for (String currKey : desc.getProps().keySet()) {
- if (!tableParams.containsKey(currKey)) {
- String errorMsg =
- "The following property " + currKey +
- " does not exist in " + tab.getTableName();
- throw new SemanticException(
- ErrorMsg.ALTER_TBL_UNSET_NON_EXIST_PROPERTY.getMsg(errorMsg));
- }
- }
- }*/
}
}
@@ -2072,8 +1962,8 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
throw new SemanticException("Owner name can't be null in alter table set owner command");
}
- AlterTableDesc alterTblDesc = new AlterTableDesc(tableName, ownerPrincipal);
- rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), alterTblDesc), conf));
+ AlterTableSetOwnerDesc alterTblDesc = new AlterTableSetOwnerDesc(tableName, ownerPrincipal);
+ rootTasks.add(TaskFactory.get(new DDLWork2(getInputs(), getOutputs(), alterTblDesc), conf));
}
private void analyzeAlterTableLocation(ASTNode ast, String tableName, Map<String, String> partSpec)
@@ -2096,14 +1986,14 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
setAcidDdlDesc(alterTblDesc);
}
- addInputsOutputsAlterTable(tableName, partSpec, alterTblDesc, AlterTableTypes.SET_LOCATION, false);
+ addInputsOutputsAlterTable(tableName, partSpec, alterTblDesc, AlterTableType.ALTERLOCATION, false);
rootTasks.add(TaskFactory.get(new DDLWork2(getInputs(), getOutputs(), alterTblDesc)));
}
private void analyzeAlterTablePartMergeFiles(ASTNode ast,
String tableName, HashMap<String, String> partSpec)
throws SemanticException {
- AlterTablePartMergeFilesDesc mergeDesc = new AlterTablePartMergeFilesDesc(
+ AlterTableConcatenateDesc mergeDesc = new AlterTableConcatenateDesc(
tableName, partSpec);
List<Path> inputDir = new ArrayList<Path>();
@@ -2122,10 +2012,9 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
boolean isBlocking = !HiveConf.getBoolVar(conf,
ConfVars.TRANSACTIONAL_CONCATENATE_NOBLOCK, false);
- AlterTableSimpleDesc desc = new AlterTableSimpleDesc(
- tableName, newPartSpec, "MAJOR", isBlocking);
+ AlterTableCompactDesc desc = new AlterTableCompactDesc(tableName, newPartSpec, "MAJOR", isBlocking, null);
- rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc)));
+ rootTasks.add(TaskFactory.get(new DDLWork2(getInputs(), getOutputs(), desc)));
return;
}
mergeDesc.setTableDesc(Utilities.getTableDesc(tblObj));
@@ -2205,8 +2094,8 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
mergeDesc.setLbCtx(lbCtx);
- addInputsOutputsAlterTable(tableName, partSpec, AlterTableTypes.MERGEFILES);
- DDLWork ddlWork = new DDLWork(getInputs(), getOutputs(), mergeDesc);
+ addInputsOutputsAlterTable(tableName, partSpec, null, AlterTableType.MERGEFILES, false);
+ DDLWork2 ddlWork = new DDLWork2(getInputs(), getOutputs(), mergeDesc);
ddlWork.setNeedLock(true);
Task<?> mergeTask = TaskFactory.get(ddlWork);
TableDesc tblDesc = Utilities.getTableDesc(tblObj);
@@ -2312,11 +2201,8 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
break;
}
}
- AlterTableSimpleDesc desc = new AlterTableSimpleDesc(
- tableName, newPartSpec, type, isBlocking);
- desc.setProps(mapProp);
-
- rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc)));
+ AlterTableCompactDesc desc = new AlterTableCompactDesc(tableName, newPartSpec, type, isBlocking, mapProp);
+ rootTasks.add(TaskFactory.get(new DDLWork2(getInputs(), getOutputs(), desc)));
}
private void analyzeAlterTableDropConstraint(ASTNode ast, String tableName)
@@ -3251,14 +3137,13 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
String sourceName = getDotName(source);
String targetName = getDotName(target);
- AlterTableDesc alterTblDesc = new AlterTableDesc(sourceName, targetName, expectView, null);
+ AlterTableRenameDesc alterTblDesc = new AlterTableRenameDesc(sourceName, null, expectView, targetName);
Table table = getTable(sourceName, true);
if (AcidUtils.isTransactionalTable(table)) {
setAcidDdlDesc(alterTblDesc);
}
- addInputsOutputsAlterTable(sourceName, null, alterTblDesc);
- rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
- alterTblDesc)));
+ addInputsOutputsAlterTable(sourceName, null, alterTblDesc, alterTblDesc.getType(), false);
+ rootTasks.add(TaskFactory.get(new DDLWork2(getInputs(), getOutputs(), alterTblDesc)));
}
private void analyzeAlterTableRenameCol(String catName, String[] qualified, ASTNode ast,
@@ -3381,7 +3266,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
private void analyzeAlterTableRenamePart(ASTNode ast, String tblName,
HashMap<String, String> oldPartSpec) throws SemanticException {
Table tab = getTable(tblName, true);
- validateAlterTableType(tab, AlterTableTypes.RENAMEPARTITION);
+ validateAlterTableType(tab, AlterTableType.RENAMEPARTITION);
Map<String, String> newPartSpec =
getValidatedPartSpec(tab, (ASTNode)ast.getChild(0), conf, false);
if (newPartSpec == null) {
@@ -3409,7 +3294,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
if (CollectionUtils.isEmpty(tab.getBucketCols())) {
throw new SemanticException(ErrorMsg.ALTER_BUCKETNUM_NONBUCKETIZED_TBL.getMsg());
}
- validateAlterTableType(tab, AlterTableTypes.INTO_BUCKETS);
+ validateAlterTableType(tab, AlterTableType.INTO_BUCKETS);
inputs.add(new ReadEntity(tab));
int numberOfBuckets = Integer.parseInt(ast.getChild(0).getText());
@@ -3502,7 +3387,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
return; // nothing to do
}
- validateAlterTableType(tab, AlterTableTypes.DROPPARTITION, expectView);
+ validateAlterTableType(tab, AlterTableType.DROPPARTITION, expectView);
ReadEntity re = new ReadEntity(tab);
re.noLockNeeded();
inputs.add(re);
@@ -3523,7 +3408,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
inputs.add(new ReadEntity(tab));
// validate the DDL is a valid operation on the table.
- validateAlterTableType(tab, AlterTableTypes.ALTERPARTITION, false);
+ validateAlterTableType(tab, AlterTableType.ALTERPARTITION, false);
// Alter table ... partition column ( column newtype) only takes one column at a time.
// It must have a column name followed with type.
@@ -3586,7 +3471,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
Table tab = getTable(qualified);
boolean isView = tab.isView();
- validateAlterTableType(tab, AlterTableTypes.ADDPARTITION, expectView);
+ validateAlterTableType(tab, AlterTableType.ADDPARTITION, expectView);
outputs.add(new WriteEntity(tab,
/*use DDL_EXCLUSIVE to cause X lock to prevent races between concurrent add partition calls
with IF NOT EXISTS. w/o this 2 concurrent calls to add the same partition may both add
@@ -3768,27 +3653,21 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
throws SemanticException {
Table tab = getTable(qualified);
- validateAlterTableType(tab, AlterTableTypes.TOUCH);
+ validateAlterTableType(tab, AlterTableType.TOUCH);
inputs.add(new ReadEntity(tab));
// partition name to value
List<Map<String, String>> partSpecs = getPartitionSpecs(tab, ast);
- if (partSpecs.size() == 0) {
- AlterTableSimpleDesc touchDesc = new AlterTableSimpleDesc(
- getDotName(qualified), null,
- AlterTableDesc.AlterTableTypes.TOUCH);
+ if (partSpecs.isEmpty()) {
+ AlterTableTouchDesc touchDesc = new AlterTableTouchDesc(getDotName(qualified), null);
outputs.add(new WriteEntity(tab, WriteEntity.WriteType.DDL_NO_LOCK));
- rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
- touchDesc)));
+ rootTasks.add(TaskFactory.get(new DDLWork2(getInputs(), getOutputs(), touchDesc)));
} else {
addTablePartsOutputs(tab, partSpecs, WriteEntity.WriteType.DDL_NO_LOCK);
for (Map<String, String> partSpec : partSpecs) {
- AlterTableSimpleDesc touchDesc = new AlterTableSimpleDesc(
- getDotName(qualified), partSpec,
- AlterTableDesc.AlterTableTypes.TOUCH);
- rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
- touchDesc)));
+ AlterTableTouchDesc touchDesc = new AlterTableTouchDesc(getDotName(qualified), partSpec);
+ rootTasks.add(TaskFactory.get(new DDLWork2(getInputs(), getOutputs(), touchDesc)));
}
}
}
@@ -3805,7 +3684,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
List<Map<String, String>> partSpecs = getPartitionSpecs(tab, ast);
addTablePartsOutputs(tab, partSpecs, true, WriteEntity.WriteType.DDL_NO_LOCK);
- validateAlterTableType(tab, AlterTableTypes.ARCHIVE);
+ validateAlterTableType(tab, AlterTableType.ARCHIVE);
inputs.add(new ReadEntity(tab));
if (partSpecs.size() > 1) {
@@ -3823,12 +3702,13 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
} catch (HiveException e) {
throw new SemanticException(e.getMessage(), e);
}
- AlterTableSimpleDesc archiveDesc = new AlterTableSimpleDesc(
- getDotName(qualified), partSpec,
- (isUnArchive ? AlterTableTypes.UNARCHIVE : AlterTableTypes.ARCHIVE));
- rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
- archiveDesc)));
-
+ DDLDesc archiveDesc = null;
+ if (isUnArchive) {
+ archiveDesc = new AlterTableUnarchiveDesc(getDotName(qualified), partSpec);
+ } else {
+ archiveDesc = new AlterTableArchiveDesc(getDotName(qualified), partSpec);
+ }
+ rootTasks.add(TaskFactory.get(new DDLWork2(getInputs(), getOutputs(), archiveDesc)));
}
/**
@@ -4209,7 +4089,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
inputs.add(new ReadEntity(tab));
outputs.add(new WriteEntity(tab, WriteEntity.WriteType.DDL_EXCLUSIVE));
- validateAlterTableType(tab, AlterTableTypes.SKEWED_BY);
+ validateAlterTableType(tab, AlterTableType.SKEWED_BY);
String tableName = getDotName(qualified);
if (ast.getChildCount() == 0) {
@@ -4346,7 +4226,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
}
}
AlterTableSetSkewedLocationDesc alterTblDesc = new AlterTableSetSkewedLocationDesc(tableName, partSpec, locations);
- addInputsOutputsAlterTable(tableName, partSpec, alterTblDesc, AlterTableTypes.SET_SKEWED_LOCATION, false);
+ addInputsOutputsAlterTable(tableName, partSpec, alterTblDesc, AlterTableType.SET_SKEWED_LOCATION, false);
rootTasks.add(TaskFactory.get(new DDLWork2(getInputs(), getOutputs(), alterTblDesc)));
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 05896cc..1a2a1e8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -103,6 +103,7 @@ import org.apache.hadoop.hive.ql.cache.results.QueryResultsCache;
import org.apache.hadoop.hive.ql.ddl.DDLWork2;
import org.apache.hadoop.hive.ql.ddl.table.creation.CreateTableDesc;
import org.apache.hadoop.hive.ql.ddl.table.creation.CreateTableLikeDesc;
+import org.apache.hadoop.hive.ql.ddl.table.misc.AlterTableUnsetPropertiesDesc;
import org.apache.hadoop.hive.ql.ddl.table.misc.PreInsertTableDesc;
import org.apache.hadoop.hive.ql.ddl.view.CreateViewDesc;
import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator;
@@ -195,8 +196,6 @@ import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec;
import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowSpec;
import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowType;
import org.apache.hadoop.hive.ql.plan.AggregationDesc;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
import org.apache.hadoop.hive.ql.plan.DDLWork;
import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
@@ -6957,13 +6956,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
private void setStatsForNonNativeTable(String dbName, String tableName) throws SemanticException {
String qTableName = DDLSemanticAnalyzer.getDotName(new String[] { dbName,
tableName });
- AlterTableDesc alterTblDesc = new AlterTableDesc(AlterTableTypes.DROPPROPS, null, false);
HashMap<String, String> mapProp = new HashMap<>();
mapProp.put(StatsSetupConst.COLUMN_STATS_ACCURATE, null);
- alterTblDesc.setOldName(qTableName);
- alterTblDesc.setProps(mapProp);
- alterTblDesc.setDropIfExists(true);
- this.rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), alterTblDesc)));
+ AlterTableUnsetPropertiesDesc alterTblDesc = new AlterTableUnsetPropertiesDesc(qTableName, null, null, false,
+ mapProp, false, null);
+ this.rootTasks.add(TaskFactory.get(new DDLWork2(getInputs(), getOutputs(), alterTblDesc)));
}
@@ -12579,7 +12576,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
rootTasks.stream()
.filter(task -> task.getWork() instanceof DDLWork2)
.map(task -> (DDLWork2) task.getWork())
- .filter(ddlWork -> ddlWork.getDDLDesc() != null)
+ .filter(ddlWork -> ddlWork.getDDLDesc() instanceof PreInsertTableDesc)
.map(ddlWork -> (PreInsertTableDesc)ddlWork.getDDLDesc())
.map(ddlPreInsertTask -> new InsertCommitHookDesc(ddlPreInsertTask.getTable(),
ddlPreInsertTask.isOverwrite()))
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenameTableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenameTableHandler.java
index 5455042..230650b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenameTableHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenameTableHandler.java
@@ -19,13 +19,13 @@ package org.apache.hadoop.hive.ql.parse.repl.load.message;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage;
+import org.apache.hadoop.hive.ql.ddl.DDLWork2;
+import org.apache.hadoop.hive.ql.ddl.table.misc.AlterTableRenameDesc;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc;
-import org.apache.hadoop.hive.ql.plan.DDLWork;
import org.apache.hadoop.hive.ql.stats.StatsUtils;
import java.io.Serializable;
@@ -61,11 +61,10 @@ public class RenameTableHandler extends AbstractMessageHandler {
if (ReplUtils.isTableMigratingToTransactional(context.hiveConf, tableObjAfter)) {
replicationSpec.setMigratingToTxnTable();
}
- AlterTableDesc renameTableDesc = new AlterTableDesc(
- oldName, newName, false, replicationSpec);
+ AlterTableRenameDesc renameTableDesc = new AlterTableRenameDesc(oldName, replicationSpec, false, newName);
renameTableDesc.setWriteId(msg.getWriteId());
- Task<DDLWork> renameTableTask = TaskFactory.get(
- new DDLWork(readEntitySet, writeEntitySet, renameTableDesc), context.hiveConf);
+ Task<DDLWork2> renameTableTask = TaskFactory.get(
+ new DDLWork2(readEntitySet, writeEntitySet, renameTableDesc), context.hiveConf);
context.log.debug("Added rename table task : {}:{}->{}",
renameTableTask.getId(), oldName, newName);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
deleted file mode 100644
index 9aa7e73..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
+++ /dev/null
@@ -1,406 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.plan;
-
-import org.apache.hadoop.hive.ql.io.AcidUtils;
-
-import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
-import org.apache.hadoop.hive.ql.ddl.privilege.PrincipalDesc;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
-import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.Explain.Level;
-import com.google.common.collect.ImmutableList;
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * AlterTableDesc.
- *
- */
-@Explain(displayName = "Alter Table", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
-public class AlterTableDesc extends DDLDesc implements Serializable, DDLDesc.DDLDescWithWriteId {
- private static final long serialVersionUID = 1L;
-
- /**
- * alterTableTypes.
- *
- */
- public static enum AlterTableTypes {
- RENAME("rename"), ADD_COLUMNS("add columns"), REPLACE_COLUMNS("replace columns"),
- ADDPROPS("add props"), DROPPROPS("drop props"), SET_SERDE("set serde"), SET_SERDE_PROPS("set serde props"),
- SET_FILE_FORMAT("add fileformat"), CLUSTERED_BY("clustered by"), NOT_SORTED("not sorted"),
- NOT_CLUSTERED("not clustered"),
- RENAME_COLUMN("rename column"), ADDPARTITION("add partition"), TOUCH("touch"), ARCHIVE("archieve"),
- UNARCHIVE("unarchieve"), SET_LOCATION("set location"),
- DROPPARTITION("drop partition"),
- RENAMEPARTITION("rename partition"), // Note: used in RenamePartitionDesc, not here.
- SKEWED_BY("skewed by"), NOT_SKEWED("not skewed"),
- SET_SKEWED_LOCATION("alter skew location"), INTO_BUCKETS("alter bucket number"),
- ALTERPARTITION("alter partition"), // Note: this is never used in AlterTableDesc.
- COMPACT("compact"),
- TRUNCATE("truncate"), MERGEFILES("merge files"), DROP_CONSTRAINT("drop constraint"),
- ADD_CONSTRAINT("add constraint"),
- UPDATE_COLUMNS("update columns"), OWNER("set owner"),
- UPDATESTATS("update stats"); // Note: used in ColumnStatsUpdateWork, not here.
- ;
-
- private final String name;
- private AlterTableTypes(String name) { this.name = name; }
- public String getName() { return name; }
-
- public static final List<AlterTableTypes> nonNativeTableAllowedTypes =
- ImmutableList.of(ADDPROPS, DROPPROPS, ADD_COLUMNS);
- }
-
- public static enum ProtectModeType {
- NO_DROP, OFFLINE, READ_ONLY, NO_DROP_CASCADE
- }
-
- public static final Set<AlterTableTypes> alterTableTypesWithPartialSpec =
- new HashSet<AlterTableDesc.AlterTableTypes>();
-
- static {
- alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.ADD_COLUMNS);
- alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.REPLACE_COLUMNS);
- alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.RENAME_COLUMN);
- alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.ADDPROPS);
- alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.DROPPROPS);
- alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.SET_SERDE);
- alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.SET_SERDE_PROPS);
- alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.SET_FILE_FORMAT);
- }
-
- AlterTableTypes op;
- String oldName;
- String newName;
- Map<String, String> props;
-
- boolean expectView;
- HashMap<String, String> partSpec;
- boolean protectModeEnable;
- ProtectModeType protectModeType;
- boolean isTurnOffSkewed = false;
- boolean isDropIfExists = false;
- boolean isCascade = false;
- EnvironmentContext environmentContext;
- ReplicationSpec replicationSpec;
- private Long writeId = null;
- PrincipalDesc ownerPrincipal;
- private boolean isExplicitStatsUpdate, isFullAcidConversion;
-
- public AlterTableDesc() {
- }
-
- /**
- * @param oldName
- * old name of the table
- * @param newName
- * new name of the table
- * @param expectView
- * Flag to denote if current table can be a view
- * @param replicationSpec
- * Replication specification with current event ID
- * @throws SemanticException
- */
- public AlterTableDesc(String oldName, String newName, boolean expectView, ReplicationSpec replicationSpec) throws SemanticException {
- op = AlterTableTypes.RENAME;
- setOldName(oldName);
- this.newName = newName;
- this.expectView = expectView;
- this.replicationSpec = replicationSpec;
- }
-
- /**
- * @param alterType
- * type of alter op
- * @param replicationSpec
- * Replication specification with current event ID
- */
- public AlterTableDesc(AlterTableTypes alterType, ReplicationSpec replicationSpec) {
- op = alterType;
- this.replicationSpec = replicationSpec;
- }
-
- /**
- * @param alterType
- * type of alter op
- */
- public AlterTableDesc(AlterTableTypes alterType) {
- this(alterType, null, false);
- }
-
- /**
- * @param alterType
- * type of alter op
- * @param expectView
- * Flag to denote if current table can be a view
- * @param partSpec
- * Partition specifier with map of key and values.
- */
- public AlterTableDesc(AlterTableTypes alterType, HashMap<String, String> partSpec, boolean expectView) {
- op = alterType;
- this.partSpec = partSpec;
- this.expectView = expectView;
- }
-
- public AlterTableDesc(String tableName, PrincipalDesc ownerPrincipal) {
- op = AlterTableTypes.OWNER;
- this.oldName = tableName;
- this.ownerPrincipal = ownerPrincipal;
- }
-
- /**
- * @param ownerPrincipal the owner principal of the table
- */
- public void setOwnerPrincipal(PrincipalDesc ownerPrincipal) {
- this.ownerPrincipal = ownerPrincipal;
- }
-
- @Explain(displayName="owner")
- public PrincipalDesc getOwnerPrincipal() {
- return this.ownerPrincipal;
- }
-
- @Explain(displayName = "type", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
- public String getAlterTableTypeString() {
- return op.getName();
- }
-
- /**
- * @return the old name of the table
- */
- @Explain(displayName = "old name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
- public String getOldName() {
- return oldName;
- }
-
- /**
- * @param oldName
- * the oldName to set
- */
- public void setOldName(String oldName) throws SemanticException {
- // Make sure we qualify the name from the outset so there's no ambiguity.
- this.oldName = String.join(".", Utilities.getDbTableName(oldName));
- }
-
- /**
- * @return the newName
- */
- @Explain(displayName = "new name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
- public String getNewName() {
- return newName;
- }
-
- /**
- * @param newName
- * the newName to set
- */
- public void setNewName(String newName) {
- this.newName = newName;
- }
-
- /**
- * @return the op
- */
- public AlterTableTypes getOp() {
- return op;
- }
-
- /**
- * @param op
- * the op to set
- */
- public void setOp(AlterTableTypes op) {
- this.op = op;
- }
-
- /**
- * @return the props
- */
- @Explain(displayName = "properties")
- public Map<String, String> getProps() {
- return props;
- }
-
- /**
- * @param props
- * the props to set
- */
- public void setProps(Map<String, String> props) {
- this.props = props;
- }
-
- /**
- * @return whether to expect a view being altered
- */
- public boolean getExpectView() {
- return expectView;
- }
-
- /**
- * @param expectView
- * set whether to expect a view being altered
- */
- public void setExpectView(boolean expectView) {
- this.expectView = expectView;
- }
-
- /**
- * @return part specification
- */
- public HashMap<String, String> getPartSpec() {
- return partSpec;
- }
-
- /**
- * @param partSpec
- */
- public void setPartSpec(HashMap<String, String> partSpec) {
- this.partSpec = partSpec;
- }
-
- public boolean isProtectModeEnable() {
- return protectModeEnable;
- }
-
- public void setProtectModeEnable(boolean protectModeEnable) {
- this.protectModeEnable = protectModeEnable;
- }
-
- public ProtectModeType getProtectModeType() {
- return protectModeType;
- }
-
- public void setProtectModeType(ProtectModeType protectModeType) {
- this.protectModeType = protectModeType;
- }
-
- /**
- * @return the turnOffSkewed
- */
- public boolean isTurnOffSkewed() {
- return isTurnOffSkewed;
- }
-
- /**
- * @param turnOffSkewed the turnOffSkewed to set
- */
- public void setTurnOffSkewed(boolean turnOffSkewed) {
- this.isTurnOffSkewed = turnOffSkewed;
- }
-
- /**
- * @param isDropIfExists the isDropIfExists to set
- */
- public void setDropIfExists(boolean isDropIfExists) {
- this.isDropIfExists = isDropIfExists;
- }
-
- /**
- * @return isDropIfExists
- */
- public boolean getIsDropIfExists() {
- return isDropIfExists;
- }
-
- /**
- * @return isCascade
- */
- public boolean getIsCascade() {
- return isCascade;
- }
-
- /**
- * @param isCascade the isCascade to set
- */
- public void setIsCascade(boolean isCascade) {
- this.isCascade = isCascade;
- }
-
- public static boolean doesAlterTableTypeSupportPartialPartitionSpec(AlterTableTypes type) {
- return alterTableTypesWithPartialSpec.contains(type);
- }
-
- public EnvironmentContext getEnvironmentContext() {
- return environmentContext;
- }
-
- public void setEnvironmentContext(EnvironmentContext environmentContext) {
- this.environmentContext = environmentContext;
- }
-
- /**
- * @return what kind of replication scope this alter is running under.
- * This can result in a "ALTER IF NEWER THAN" kind of semantic
- */
- public ReplicationSpec getReplicationSpec(){ return this.replicationSpec; }
-
- @Override
- public void setWriteId(long writeId) {
- this.writeId = writeId;
- }
-
- @Override
- public String getFullTableName() {
- return getOldName();
- }
-
- @Override
- public boolean mayNeedWriteId() {
- switch (getOp()) {
- case ADDPROPS: {
- return isExplicitStatsUpdate || AcidUtils.isToInsertOnlyTable(null, getProps())
- || (AcidUtils.isTransactionalTable(getProps()) && !isFullAcidConversion);
- }
- case DROPPROPS: return isExplicitStatsUpdate;
- // The check for the following ones is performed before setting AlterTableDesc into the acid field.
- // These need write ID and stuff because they invalidate column stats.
- case RENAME_COLUMN:
- case RENAME:
- case REPLACE_COLUMNS:
- case ADD_COLUMNS:
- case SET_LOCATION:
- case UPDATE_COLUMNS: return true;
- // RENAMEPARTITION is handled in RenamePartitionDesc
- default: return false;
- }
- }
-
- public Long getWriteId() {
- return this.writeId;
- }
-
- @Override
- public String toString() {
- return this.getClass().getSimpleName() + " for " + getFullTableName();
- }
-
- public void setIsExplicitStatsUpdate(boolean b) {
- this.isExplicitStatsUpdate = b;
- }
-
- public void setIsFullAcidConversion(boolean b) {
- this.isFullAcidConversion = b;
- }
-}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableSimpleDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableSimpleDesc.java
deleted file mode 100644
index 759a14f..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableSimpleDesc.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.plan;
-
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
-
-/**
- * Contains information needed to modify a partition or a table
- */
-public class AlterTableSimpleDesc extends DDLDesc {
- private String tableName;
- private LinkedHashMap<String, String> partSpec;
- private String compactionType;
- private boolean isBlocking = false;
-
- AlterTableTypes type;
- private Map<String, String> props;
-
- public AlterTableSimpleDesc() {
- }
-
- /**
- * @param tableName
- * table containing the partition
- * @param partSpec
- */
- public AlterTableSimpleDesc(String tableName,
- Map<String, String> partSpec, AlterTableTypes type) {
- this.tableName = tableName;
- if(partSpec == null) {
- this.partSpec = null;
- } else {
- this.partSpec = new LinkedHashMap<String,String>(partSpec);
- }
- this.type = type;
- }
-
- /**
- * Constructor for ALTER TABLE ... COMPACT.
- * @param tableName name of the table to compact
- * @param partSpec partition to compact
- * @param compactionType currently supported values: 'major' and 'minor'
- */
- public AlterTableSimpleDesc(String tableName,
- LinkedHashMap<String, String> partSpec, String compactionType, boolean isBlocking) {
- type = AlterTableTypes.COMPACT;
- this.compactionType = compactionType;
- this.tableName = tableName;
- this.partSpec = partSpec;
- this.isBlocking = isBlocking;
- }
-
- public String getTableName() {
- return tableName;
- }
-
- public void setTableName(String tableName) {
- this.tableName = tableName;
- }
-
- public AlterTableDesc.AlterTableTypes getType() {
- return type;
- }
-
- public void setType(AlterTableDesc.AlterTableTypes type) {
- this.type = type;
- }
-
- public LinkedHashMap<String, String> getPartSpec() {
- return partSpec;
- }
-
- public void setPartSpec(LinkedHashMap<String, String> partSpec) {
- this.partSpec = partSpec;
- }
-
- /**
- * Get what type of compaction is being done by a ALTER TABLE ... COMPACT statement.
- * @return Compaction type, currently supported values are 'major' and 'minor'.
- */
- public String getCompactionType() {
- return compactionType;
- }
-
- /**
- * if compaction request should block until completion
- */
- public boolean isBlocking() {
- return isBlocking;
- }
-
- public Map<String, String> getProps() {
- return props;
- }
-
- public void setProps(Map<String, String> props) {
- this.props = props;
- }
-}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java
index 07feae3..639aaba 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.plan;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
-import org.apache.hadoop.hive.ql.parse.AlterTablePartMergeFilesDesc;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
import java.io.Serializable;
@@ -34,8 +33,6 @@ public class DDLWork implements Serializable {
// TODO: this can probably be replaced with much less code via dynamic dispatch and/or templates.
private InsertCommitHookDesc insertCommitHookDesc;
- private AlterTableDesc alterTblDesc;
- private AlterTableSimpleDesc alterTblSimpleDesc;
private MsckDesc msckDesc;
private ShowConfDesc showConfDesc;
@@ -52,7 +49,6 @@ public class DDLWork implements Serializable {
* List of WriteEntities that are passed to the hooks.
*/
protected HashSet<WriteEntity> outputs;
- private AlterTablePartMergeFilesDesc mergeFilesDesc;
private CacheMetadataDesc cacheMetadataDesc;
public DDLWork() {
@@ -69,28 +65,6 @@ public class DDLWork implements Serializable {
this.showConfDesc = showConfDesc;
}
- /**
- * @param alterTblDesc
- * alter table descriptor
- */
- public DDLWork(HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs,
- AlterTableDesc alterTblDesc) {
- this(inputs, outputs);
- this.alterTblDesc = alterTblDesc;
- }
-
- /**
- * @param inputs
- * @param outputs
- * @param simpleDesc
- */
- public DDLWork(HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs,
- AlterTableSimpleDesc simpleDesc) {
- this(inputs, outputs);
-
- this.alterTblSimpleDesc = simpleDesc;
- }
-
public DDLWork(HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs,
MsckDesc checkDesc) {
this(inputs, outputs);
@@ -99,12 +73,6 @@ public class DDLWork implements Serializable {
}
public DDLWork(HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs,
- AlterTablePartMergeFilesDesc mergeDesc) {
- this(inputs, outputs);
- this.mergeFilesDesc = mergeDesc;
- }
-
- public DDLWork(HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs,
CacheMetadataDesc cacheMetadataDesc) {
this(inputs, outputs);
this.cacheMetadataDesc = cacheMetadataDesc;
@@ -124,21 +92,6 @@ public class DDLWork implements Serializable {
}
/**
- * @return the alterTblDesc
- */
- @Explain(displayName = "Alter Table Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
- public AlterTableDesc getAlterTblDesc() {
- return alterTblDesc;
- }
-
- /**
- * @return information about the table/partitions we want to alter.
- */
- public AlterTableSimpleDesc getAlterTblSimpleDesc() {
- return alterTblSimpleDesc;
- }
-
- /**
* @return Metastore check description
*/
public MsckDesc getMsckDesc() {
@@ -153,13 +106,6 @@ public class DDLWork implements Serializable {
return outputs;
}
- /**
- * @return descriptor for merging files
- */
- public AlterTablePartMergeFilesDesc getMergeFilesDesc() {
- return mergeFilesDesc;
- }
-
public boolean getNeedLock() {
return needLock;
}
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestQBCompact.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestQBCompact.java
index 61d74de..19ab8e6 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestQBCompact.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestQBCompact.java
@@ -26,13 +26,13 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.ddl.DDLWork2;
+import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableCompactDesc;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.plan.AlterTableSimpleDesc;
-import org.apache.hadoop.hive.ql.plan.DDLWork;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -84,19 +84,19 @@ public class TestQBCompact {
Assert.assertEquals(1, roots.size());
}
- private AlterTableSimpleDesc parseAndAnalyzeAlterTable(String query) throws Exception {
+ private AlterTableCompactDesc parseAndAnalyzeAlterTable(String query) throws Exception {
ParseDriver hd = new ParseDriver();
ASTNode head = (ASTNode)hd.parse(query).getChild(0);
BaseSemanticAnalyzer a = SemanticAnalyzerFactory.get(queryState, head);
a.analyze(head, new Context(conf));
List<Task<?>> roots = a.getRootTasks();
Assert.assertEquals(1, roots.size());
- return ((DDLWork)roots.get(0).getWork()).getAlterTblSimpleDesc();
+ return (AlterTableCompactDesc)((DDLWork2)roots.get(0).getWork()).getDDLDesc();
}
@Test
public void testNonPartitionedTable() throws Exception {
- AlterTableSimpleDesc desc = parseAndAnalyzeAlterTable("alter table foo compact 'major'");
+ AlterTableCompactDesc desc = parseAndAnalyzeAlterTable("alter table foo compact 'major'");
Assert.assertEquals("major", desc.getCompactionType());
Assert.assertEquals("default.foo", desc.getTableName());
}
@@ -115,22 +115,22 @@ public class TestQBCompact {
@Test
public void testMajor() throws Exception {
- AlterTableSimpleDesc desc =
+ AlterTableCompactDesc desc =
parseAndAnalyzeAlterTable("alter table foo partition(ds = 'today') compact 'major'");
Assert.assertEquals("major", desc.getCompactionType());
Assert.assertEquals("default.foo", desc.getTableName());
- HashMap<String, String> parts = desc.getPartSpec();
+ Map<String, String> parts = desc.getPartitionSpec();
Assert.assertEquals(1, parts.size());
Assert.assertEquals("today", parts.get("ds"));
}
@Test
public void testMinor() throws Exception {
- AlterTableSimpleDesc desc =
+ AlterTableCompactDesc desc =
parseAndAnalyzeAlterTable("alter table foo partition(ds = 'today') compact 'minor'");
Assert.assertEquals("minor", desc.getCompactionType());
Assert.assertEquals("default.foo", desc.getTableName());
- HashMap<String, String> parts = desc.getPartSpec();
+ Map<String, String> parts = desc.getPartitionSpec();
Assert.assertEquals(1, parts.size());
Assert.assertEquals("today", parts.get("ds"));
}
diff --git a/ql/src/test/queries/clientpositive/acid_vectorization_original_tez.q b/ql/src/test/queries/clientpositive/acid_vectorization_original_tez.q
index 50fb3e9..8e3179c 100644
--- a/ql/src/test/queries/clientpositive/acid_vectorization_original_tez.q
+++ b/ql/src/test/queries/clientpositive/acid_vectorization_original_tez.q
@@ -106,6 +106,7 @@ explain select ROW__ID, count(*) from over10k_orc_bucketed_n0 group by ROW__ID h
select ROW__ID, count(*) from over10k_orc_bucketed_n0 group by ROW__ID having count(*) > 1;
-- schedule compactor
+explain alter table over10k_orc_bucketed_n0 compact 'major' WITH OVERWRITE TBLPROPERTIES ('compactor.mapreduce.map.memory.mb'='500', 'compactor.mapreduce.reduce.memory.mb'='500','compactor.mapreduce.map.memory.mb'='500', 'compactor.hive.tez.container.size'='500');
alter table over10k_orc_bucketed_n0 compact 'major' WITH OVERWRITE TBLPROPERTIES ('compactor.mapreduce.map.memory.mb'='500', 'compactor.mapreduce.reduce.memory.mb'='500','compactor.mapreduce.map.memory.mb'='500', 'compactor.hive.tez.container.size'='500');
diff --git a/ql/src/test/queries/clientpositive/archive_multi.q b/ql/src/test/queries/clientpositive/archive_multi.q
index b372ea2..9081dec 100644
--- a/ql/src/test/queries/clientpositive/archive_multi.q
+++ b/ql/src/test/queries/clientpositive/archive_multi.q
@@ -27,6 +27,7 @@ select key, value from default.srcpart where ds='2008-04-09' and hr='12';
SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col
FROM (SELECT * FROM ac_test.tstsrcpart WHERE ds='2008-04-08') subq1) subq2;
+EXPLAIN ALTER TABLE ac_test.tstsrcpart ARCHIVE PARTITION (ds='2008-04-08');
ALTER TABLE ac_test.tstsrcpart ARCHIVE PARTITION (ds='2008-04-08');
SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col
@@ -37,6 +38,7 @@ SELECT key, count(1) FROM ac_test.tstsrcpart WHERE ds='2008-04-08' AND hr='12' A
SELECT * FROM ac_test.tstsrcpart a JOIN ac_test.tstsrc b ON a.key=b.key
WHERE a.ds='2008-04-08' AND a.hr='12' AND a.key='0';
+EXPLAIN ALTER TABLE ac_test.tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08');
ALTER TABLE ac_test.tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08');
SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col
diff --git a/ql/src/test/queries/clientpositive/dbtxnmgr_compact2.q b/ql/src/test/queries/clientpositive/dbtxnmgr_compact2.q
index 6e560dd..424255c 100644
--- a/ql/src/test/queries/clientpositive/dbtxnmgr_compact2.q
+++ b/ql/src/test/queries/clientpositive/dbtxnmgr_compact2.q
@@ -8,6 +8,7 @@ alter table T1_n105 add partition (ds = 'yesterday');
alter table T1_n105 partition (ds = 'today') compact 'major';
+explain alter table T1_n105 partition (ds = 'yesterday') compact 'minor';
alter table T1_n105 partition (ds = 'yesterday') compact 'minor';
drop table T1_n105;
diff --git a/ql/src/test/queries/clientpositive/set_tblproperties.q b/ql/src/test/queries/clientpositive/set_tblproperties.q
new file mode 100644
index 0000000..48cbe7c
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/set_tblproperties.q
@@ -0,0 +1,8 @@
+CREATE TABLE t (a INT);
+
+EXPLAIN ALTER TABLE t SET TBLPROPERTIES('a'='x', 'b'='y');
+ALTER TABLE t SET TBLPROPERTIES('a'='x', 'b'='y');
+SHOW CREATE TABLE t;
+EXPLAIN ALTER TABLE t UNSET TBLPROPERTIES('a', 'b');
+ALTER TABLE t UNSET TBLPROPERTIES('a', 'b');
+SHOW CREATE TABLE t;
diff --git a/ql/src/test/queries/clientpositive/table_set_owner.q b/ql/src/test/queries/clientpositive/table_set_owner.q
new file mode 100644
index 0000000..73c9eab
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/table_set_owner.q
@@ -0,0 +1,4 @@
+SET hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.DefaultHiveAuthorizationProvider;
+CREATE TABLE t (a INT);
+EXPLAIN ALTER TABLE t SET OWNER USER user1;
+ALTER TABLE t SET OWNER USER user1;
diff --git a/ql/src/test/queries/clientpositive/touch.q b/ql/src/test/queries/clientpositive/touch.q
index 8711b77..0fb34d0 100644
--- a/ql/src/test/queries/clientpositive/touch.q
+++ b/ql/src/test/queries/clientpositive/touch.q
@@ -9,8 +9,10 @@ create table tc.tstsrcpart like default.srcpart;
insert overwrite table tc.tstsrcpart partition (ds='2008-04-08', hr='12')
select key, value from default.srcpart where ds='2008-04-08' and hr='12';
+EXPLAIN ALTER TABLE tc.tstsrc TOUCH;
ALTER TABLE tc.tstsrc TOUCH;
ALTER TABLE tc.tstsrcpart TOUCH;
+EXPLAIN ALTER TABLE tc.tstsrcpart TOUCH PARTITION (ds='2008-04-08', hr='12');
ALTER TABLE tc.tstsrcpart TOUCH PARTITION (ds='2008-04-08', hr='12');
drop table tc.tstsrc;
diff --git a/ql/src/test/results/clientnegative/alter_external_acid.q.out b/ql/src/test/results/clientnegative/alter_external_acid.q.out
index 69bba3b..51e44b8 100644
--- a/ql/src/test/results/clientnegative/alter_external_acid.q.out
+++ b/ql/src/test/results/clientnegative/alter_external_acid.q.out
@@ -10,4 +10,4 @@ PREHOOK: query: alter table acid_external set TBLPROPERTIES ('transactional'='tr
PREHOOK: type: ALTERTABLE_PROPERTIES
PREHOOK: Input: default@acid_external
PREHOOK: Output: default@acid_external
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unable to alter table. default.acid_external cannot be declared transactional because it's an external table
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Unable to alter table. default.acid_external cannot be declared transactional because it's an external table
diff --git a/ql/src/test/results/clientnegative/alter_non_native.q.out b/ql/src/test/results/clientnegative/alter_non_native.q.out
index 110c9f0..bd8fb4f 100644
--- a/ql/src/test/results/clientnegative/alter_non_native.q.out
+++ b/ql/src/test/results/clientnegative/alter_non_native.q.out
@@ -8,4 +8,4 @@ STORED BY 'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler'
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@non_native1
-FAILED: SemanticException [Error 10134]: ALTER TABLE can only be used for [ADDPROPS, DROPPROPS, ADD_COLUMNS] to a non-native table non_native1
+FAILED: SemanticException [Error 10134]: ALTER TABLE can only be used for [ADDPROPS, DROPPROPS, ADDCOLS] to a non-native table non_native1
diff --git a/ql/src/test/results/clientnegative/alter_table_wrong_db.q.out b/ql/src/test/results/clientnegative/alter_table_wrong_db.q.out
index 641e09f..eca572b 100644
--- a/ql/src/test/results/clientnegative/alter_table_wrong_db.q.out
+++ b/ql/src/test/results/clientnegative/alter_table_wrong_db.q.out
@@ -22,4 +22,4 @@ PREHOOK: query: alter table bad_rename1.rename1 rename to bad_db_notexists.renam
PREHOOK: type: ALTERTABLE_RENAME
PREHOOK: Input: bad_rename1@rename1
PREHOOK: Output: bad_rename1@rename1
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unable to alter table. Unable to change partition or table. Object bad_db_notexists does not exist. Check metastore logs for detailed stack.
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Unable to alter table. Unable to change partition or table. Object bad_db_notexists does not exist. Check metastore logs for detailed stack.
diff --git a/ql/src/test/results/clientnegative/archive1.q.out b/ql/src/test/results/clientnegative/archive1.q.out
index b530855..d52ac0e 100644
--- a/ql/src/test/results/clientnegative/archive1.q.out
+++ b/ql/src/test/results/clientnegative/archive1.q.out
@@ -32,4 +32,4 @@ PREHOOK: query: ALTER TABLE srcpart_archived ARCHIVE PARTITION (ds='2008-04-08',
PREHOOK: type: ALTERTABLE_ARCHIVE
PREHOOK: Input: default@srcpart_archived
PREHOOK: Output: default@srcpart_archived@ds=2008-04-08/hr=12
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Partition(s) already archived
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Partition(s) already archived
diff --git a/ql/src/test/results/clientnegative/archive2.q.out b/ql/src/test/results/clientnegative/archive2.q.out
index 471df78..75d823e 100644
--- a/ql/src/test/results/clientnegative/archive2.q.out
+++ b/ql/src/test/results/clientnegative/archive2.q.out
@@ -28,4 +28,4 @@ PREHOOK: query: ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08', hr=
PREHOOK: type: ALTERTABLE_UNARCHIVE
PREHOOK: Input: default@tstsrcpart
PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Partition ds=2008-04-08/hr=12 is not archived.
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Partition ds=2008-04-08/hr=12 is not archived.
diff --git a/ql/src/test/results/clientnegative/archive_multi1.q.out b/ql/src/test/results/clientnegative/archive_multi1.q.out
index 3fed11e..c5f85cc 100644
--- a/ql/src/test/results/clientnegative/archive_multi1.q.out
+++ b/ql/src/test/results/clientnegative/archive_multi1.q.out
@@ -49,4 +49,4 @@ PREHOOK: type: ALTERTABLE_ARCHIVE
PREHOOK: Input: default@tstsrcpart
PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Partition(s) already archived
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Partition(s) already archived
diff --git a/ql/src/test/results/clientnegative/archive_multi2.q.out b/ql/src/test/results/clientnegative/archive_multi2.q.out
index 134a342..460b781 100644
--- a/ql/src/test/results/clientnegative/archive_multi2.q.out
+++ b/ql/src/test/results/clientnegative/archive_multi2.q.out
@@ -42,4 +42,4 @@ PREHOOK: query: ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08', hr=
PREHOOK: type: ALTERTABLE_UNARCHIVE
PREHOOK: Input: default@tstsrcpart
PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Partition ds=2008-04-08/hr=12 is not archived.
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Partition ds=2008-04-08/hr=12 is not archived.
diff --git a/ql/src/test/results/clientnegative/archive_multi3.q.out b/ql/src/test/results/clientnegative/archive_multi3.q.out
index a4cd806..0095bbe 100644
--- a/ql/src/test/results/clientnegative/archive_multi3.q.out
+++ b/ql/src/test/results/clientnegative/archive_multi3.q.out
@@ -47,4 +47,4 @@ PREHOOK: type: ALTERTABLE_ARCHIVE
PREHOOK: Input: default@tstsrcpart
PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Conflict with existing archive ds=2008-04-08/hr=12
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Conflict with existing archive ds=2008-04-08/hr=12
diff --git a/ql/src/test/results/clientnegative/archive_multi4.q.out b/ql/src/test/results/clientnegative/archive_multi4.q.out
index 4221d25..8413e83 100644
--- a/ql/src/test/results/clientnegative/archive_multi4.q.out
+++ b/ql/src/test/results/clientnegative/archive_multi4.q.out
@@ -48,4 +48,4 @@ PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='1
PREHOOK: type: ALTERTABLE_ARCHIVE
PREHOOK: Input: default@tstsrcpart
PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Conflict with existing archive ds=2008-04-08
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Conflict with existing archive ds=2008-04-08
diff --git a/ql/src/test/results/clientnegative/archive_multi5.q.out b/ql/src/test/results/clientnegative/archive_multi5.q.out
index 315fbe4..36a7830 100644
--- a/ql/src/test/results/clientnegative/archive_multi5.q.out
+++ b/ql/src/test/results/clientnegative/archive_multi5.q.out
@@ -47,4 +47,4 @@ PREHOOK: type: ALTERTABLE_UNARCHIVE
PREHOOK: Input: default@tstsrcpart
PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Partition ds=2008-04-08/hr=11 is not archived.
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Partition ds=2008-04-08/hr=11 is not archived.
diff --git a/ql/src/test/results/clientnegative/archive_multi6.q.out b/ql/src/test/results/clientnegative/archive_multi6.q.out
index e73ddd1..2046db9 100644
--- a/ql/src/test/results/clientnegative/archive_multi6.q.out
+++ b/ql/src/test/results/clientnegative/archive_multi6.q.out
@@ -48,4 +48,4 @@ PREHOOK: query: ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08', hr=
PREHOOK: type: ALTERTABLE_UNARCHIVE
PREHOOK: Input: default@tstsrcpart
PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Partition ds=2008-04-08/hr=12 is archived at level 1, and given partspec only has 2 specs.
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Partition ds=2008-04-08/hr=12 is archived at level 1, and given partspec only has 2 specs.
diff --git a/ql/src/test/results/clientnegative/compact_non_acid_table.q.out b/ql/src/test/results/clientnegative/compact_non_acid_table.q.out
index eab9e19..2e1f197 100644
--- a/ql/src/test/results/clientnegative/compact_non_acid_table.q.out
+++ b/ql/src/test/results/clientnegative/compact_non_acid_table.q.out
@@ -8,4 +8,4 @@ POSTHOOK: Output: database:default
POSTHOOK: Output: default@not_an_acid_table
PREHOOK: query: alter table not_an_acid_table compact 'major'
PREHOOK: type: ALTERTABLE_COMPACT
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Compaction is not allowed on non-ACID table default.not_an_acid_table
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Compaction is not allowed on non-ACID table default.not_an_acid_table
diff --git a/ql/src/test/results/clientnegative/mm_convert.q.out b/ql/src/test/results/clientnegative/mm_convert.q.out
index 496f8b5..2463a56 100644
--- a/ql/src/test/results/clientnegative/mm_convert.q.out
+++ b/ql/src/test/results/clientnegative/mm_convert.q.out
@@ -14,4 +14,4 @@ PREHOOK: query: alter table convert_mm unset tblproperties('transactional_proper
PREHOOK: type: ALTERTABLE_PROPERTIES
PREHOOK: Input: default@convert_mm
PREHOOK: Output: default@convert_mm
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Cannot convert an ACID table to non-ACID
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Cannot convert an ACID table to non-ACID
diff --git a/ql/src/test/results/clientnegative/strict_managed_tables2.q.out b/ql/src/test/results/clientnegative/strict_managed_tables2.q.out
index 85753a4..286698d 100644
--- a/ql/src/test/results/clientnegative/strict_managed_tables2.q.out
+++ b/ql/src/test/results/clientnegative/strict_managed_tables2.q.out
@@ -10,4 +10,4 @@ PREHOOK: query: alter table strict_managed_tables2_tab1 set tblproperties ('EXTE
PREHOOK: type: ALTERTABLE_PROPERTIES
PREHOOK: Input: default@strict_managed_tables2_tab1
PREHOOK: Output: default@strict_managed_tables2_tab1
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unable to alter table. Table default.strict_managed_tables2_tab1 failed strict managed table checks due to the following reason: Table is marked as a managed table but is not transactional.
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Unable to alter table. Table default.strict_managed_tables2_tab1 failed strict managed table checks due to the following reason: Table is marked as a managed table but is not transactional.
diff --git a/ql/src/test/results/clientnegative/strict_managed_tables3.q.out b/ql/src/test/results/clientnegative/strict_managed_tables3.q.out
index 7de8a80..a5a82e0 100644
--- a/ql/src/test/results/clientnegative/strict_managed_tables3.q.out
+++ b/ql/src/test/results/clientnegative/strict_managed_tables3.q.out
@@ -10,4 +10,4 @@ PREHOOK: query: alter table strict_managed_tables3_tab1 unset tblproperties ('EX
PREHOOK: type: ALTERTABLE_PROPERTIES
PREHOOK: Input: default@strict_managed_tables3_tab1
PREHOOK: Output: default@strict_managed_tables3_tab1
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unable to alter table. Table default.strict_managed_tables3_tab1 failed strict managed table checks due to the following reason: Table is marked as a managed table but is not transactional.
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Unable to alter table. Table default.strict_managed_tables3_tab1 failed strict managed table checks due to the following reason: Table is marked as a managed table but is not transactional.
diff --git a/ql/src/test/results/clientnegative/temp_table_rename.q.out b/ql/src/test/results/clientnegative/temp_table_rename.q.out
index 0956aba..4c24efd 100644
--- a/ql/src/test/results/clientnegative/temp_table_rename.q.out
+++ b/ql/src/test/results/clientnegative/temp_table_rename.q.out
@@ -18,4 +18,4 @@ PREHOOK: query: alter table tmp2 rename to tmp1
PREHOOK: type: ALTERTABLE_RENAME
PREHOOK: Input: default@tmp2
PREHOOK: Output: default@tmp2
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unable to alter table. Cannot rename temporary table to tmp1 - temporary table already exists with the same name
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Unable to alter table. Cannot rename temporary table to tmp1 - temporary table already exists with the same name
diff --git a/ql/src/test/results/clientnegative/touch1.q.out b/ql/src/test/results/clientnegative/touch1.q.out
index cc1e52e..342a505 100644
--- a/ql/src/test/results/clientnegative/touch1.q.out
+++ b/ql/src/test/results/clientnegative/touch1.q.out
@@ -1,4 +1,4 @@
PREHOOK: query: ALTER TABLE srcpart TOUCH PARTITION (ds='2008-04-08', hr='13')
PREHOOK: type: ALTERTABLE_TOUCH
PREHOOK: Input: default@srcpart
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Specified partition does not exist
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Specified partition does not exist
diff --git a/ql/src/test/results/clientpositive/alter_rename_table.q.out b/ql/src/test/results/clientpositive/alter_rename_table.q.out
index 32919ea..dd65695 100644
--- a/ql/src/test/results/clientpositive/alter_rename_table.q.out
+++ b/ql/src/test/results/clientpositive/alter_rename_table.q.out
@@ -129,11 +129,9 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
- Alter Table Operator:
- Alter Table
- type: rename
- new name: target.src
- old name: source.src
+ Rename Table
+ new table name: target.src
+ table name: source.src
PREHOOK: query: ALTER TABLE source.src RENAME TO target.src
PREHOOK: type: ALTERTABLE_RENAME
@@ -177,11 +175,9 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
- Alter Table Operator:
- Alter Table
- type: rename
- new name: target.srcpart
- old name: source.srcpart
+ Rename Table
+ new table name: target.srcpart
+ table name: source.srcpart
PREHOOK: query: ALTER TABLE source.srcpart RENAME TO target.srcpart
PREHOOK: type: ALTERTABLE_RENAME
diff --git a/ql/src/test/results/clientpositive/archive_multi.q.out b/ql/src/test/results/clientpositive/archive_multi.q.out
index 5222c33..eea3a6e 100644
--- a/ql/src/test/results/clientpositive/archive_multi.q.out
+++ b/ql/src/test/results/clientpositive/archive_multi.q.out
@@ -101,6 +101,26 @@ POSTHOOK: Input: ac_test@tstsrcpart@ds=2008-04-08/hr=11
POSTHOOK: Input: ac_test@tstsrcpart@ds=2008-04-08/hr=12
#### A masked pattern was here ####
48479881068
+PREHOOK: query: EXPLAIN ALTER TABLE ac_test.tstsrcpart ARCHIVE PARTITION (ds='2008-04-08')
+PREHOOK: type: ALTERTABLE_ARCHIVE
+PREHOOK: Input: ac_test@tstsrcpart
+PREHOOK: Output: ac_test@tstsrcpart@ds=2008-04-08/hr=11
+PREHOOK: Output: ac_test@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: EXPLAIN ALTER TABLE ac_test.tstsrcpart ARCHIVE PARTITION (ds='2008-04-08')
+POSTHOOK: type: ALTERTABLE_ARCHIVE
+POSTHOOK: Input: ac_test@tstsrcpart
+POSTHOOK: Output: ac_test@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: Output: ac_test@tstsrcpart@ds=2008-04-08/hr=12
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Archive
+ partition spec:
+ ds 2008-04-08
+ table name: ac_test.tstsrcpart
+
PREHOOK: query: ALTER TABLE ac_test.tstsrcpart ARCHIVE PARTITION (ds='2008-04-08')
PREHOOK: type: ALTERTABLE_ARCHIVE
PREHOOK: Input: ac_test@tstsrcpart
@@ -161,6 +181,26 @@ POSTHOOK: Input: ac_test@tstsrcpart@ds=2008-04-08/hr=12
0 val_0 2008-04-08 12 0 val_0
0 val_0 2008-04-08 12 0 val_0
0 val_0 2008-04-08 12 0 val_0
+PREHOOK: query: EXPLAIN ALTER TABLE ac_test.tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08')
+PREHOOK: type: ALTERTABLE_UNARCHIVE
+PREHOOK: Input: ac_test@tstsrcpart
+PREHOOK: Output: ac_test@tstsrcpart@ds=2008-04-08/hr=11
+PREHOOK: Output: ac_test@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: EXPLAIN ALTER TABLE ac_test.tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08')
+POSTHOOK: type: ALTERTABLE_UNARCHIVE
+POSTHOOK: Input: ac_test@tstsrcpart
+POSTHOOK: Output: ac_test@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: Output: ac_test@tstsrcpart@ds=2008-04-08/hr=12
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Unarchive
+ partition spec:
+ ds 2008-04-08
+ table name: ac_test.tstsrcpart
+
PREHOOK: query: ALTER TABLE ac_test.tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08')
PREHOOK: type: ALTERTABLE_UNARCHIVE
PREHOOK: Input: ac_test@tstsrcpart
diff --git a/ql/src/test/results/clientpositive/dbtxnmgr_compact2.q.out b/ql/src/test/results/clientpositive/dbtxnmgr_compact2.q.out
index b749846..9d744e2 100644
--- a/ql/src/test/results/clientpositive/dbtxnmgr_compact2.q.out
+++ b/ql/src/test/results/clientpositive/dbtxnmgr_compact2.q.out
@@ -24,6 +24,21 @@ PREHOOK: query: alter table T1_n105 partition (ds = 'today') compact 'major'
PREHOOK: type: ALTERTABLE_COMPACT
POSTHOOK: query: alter table T1_n105 partition (ds = 'today') compact 'major'
POSTHOOK: type: ALTERTABLE_COMPACT
+PREHOOK: query: explain alter table T1_n105 partition (ds = 'yesterday') compact 'minor'
+PREHOOK: type: ALTERTABLE_COMPACT
+POSTHOOK: query: explain alter table T1_n105 partition (ds = 'yesterday') compact 'minor'
+POSTHOOK: type: ALTERTABLE_COMPACT
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Compact
+ compaction type: minor
+ partition spec:
+ ds yesterday
+ table name: default.T1_n105
+
PREHOOK: query: alter table T1_n105 partition (ds = 'yesterday') compact 'minor'
PREHOOK: type: ALTERTABLE_COMPACT
POSTHOOK: query: alter table T1_n105 partition (ds = 'yesterday') compact 'minor'
diff --git a/ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out b/ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out
index 7506ca8..1401dbc 100644
--- a/ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out
+++ b/ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out
@@ -344,12 +344,10 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
- Alter Table Operator:
- Alter Table
- type: drop props
- old name: default.druid_partitioned_table
- properties:
- COLUMN_STATS_ACCURATE
+ Unset Properties
+ properties:
+ COLUMN_STATS_ACCURATE
+ table name: default.druid_partitioned_table
Stage: Stage-1
Pre-Insert task
@@ -488,12 +486,10 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
- Alter Table Operator:
- Alter Table
- type: drop props
- old name: default.druid_partitioned_table
- properties:
- COLUMN_STATS_ACCURATE
+ Unset Properties
+ properties:
+ COLUMN_STATS_ACCURATE
+ table name: default.druid_partitioned_table
Stage: Stage-1
Pre-Insert task
diff --git a/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out b/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out
index 483c9c1..08a83da 100644
--- a/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out
+++ b/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out
@@ -471,12 +471,10 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
- Alter Table Operator:
- Alter Table
- type: drop props
- old name: default.cmv_mat_view2_n0
- properties:
- COLUMN_STATS_ACCURATE
+ Unset Properties
+ properties:
+ COLUMN_STATS_ACCURATE
+ table name: default.cmv_mat_view2_n0
Stage: Stage-4
Materialized View Work
diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out
index 23dbced..f2fcf7f 100644
--- a/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out
+++ b/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out
@@ -202,7 +202,7 @@ PREHOOK: query: ALTER TABLE encrypted_db_outloc.renamed_encrypted_table_n1 RENAM
PREHOOK: type: ALTERTABLE_RENAME
PREHOOK: Input: encrypted_db_outloc@renamed_encrypted_table_n1
PREHOOK: Output: encrypted_db_outloc@renamed_encrypted_table_n1
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unable to alter table. Alter Table operation for encrypted_db_outloc.renamed_encrypted_table_n1 failed to move data due to: 'Got exception: org.apache.hadoop.ipc.RemoteException /build/ql/test/data/specified_db_location/renamed_encrypted_table_n1 can't be moved from an encryption zone.' See hive log file for details.
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Unable to alter table. Alter Table operation for encrypted_db_outloc.renamed_encrypted_table_n1 failed to move data due to: 'Got exception: org.apache.hadoop.ipc.RemoteException /build/ql/test/data/specified_db_location/renamed_encrypted_table_n1 can't be moved from an encryption zone.' See hive log file for details.
PREHOOK: query: SHOW TABLES
PREHOOK: type: SHOWTABLES
PREHOOK: Input: database:encrypted_db_outloc
diff --git a/ql/src/test/results/clientpositive/input3.q.out b/ql/src/test/results/clientpositive/input3.q.out
index c521ed6..3d82be7 100644
--- a/ql/src/test/results/clientpositive/input3.q.out
+++ b/ql/src/test/results/clientpositive/input3.q.out
@@ -91,11 +91,9 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
- Alter Table Operator:
- Alter Table
- type: rename
- new name: default.TEST3c
- old name: default.TEST3b
+ Rename Table
+ new table name: default.TEST3c
+ table name: default.TEST3b
PREHOOK: query: ALTER TABLE TEST3b RENAME TO TEST3c
PREHOOK: type: ALTERTABLE_RENAME
diff --git a/ql/src/test/results/clientpositive/llap/orc_merge10.q.out b/ql/src/test/results/clientpositive/llap/orc_merge10.q.out
index 22b4371..3af8190 100644
--- a/ql/src/test/results/clientpositive/llap/orc_merge10.q.out
+++ b/ql/src/test/results/clientpositive/llap/orc_merge10.q.out
@@ -631,6 +631,11 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
+ Concatenate
+ partition desc:
+ ds 1
+ part 0
+ table name: default.orcfile_merge1
Stage: Stage-1
Move Operator
diff --git a/ql/src/test/results/clientpositive/llap/orc_merge5.q.out b/ql/src/test/results/clientpositive/llap/orc_merge5.q.out
index 5b24776..6790cd7 100644
--- a/ql/src/test/results/clientpositive/llap/orc_merge5.q.out
+++ b/ql/src/test/results/clientpositive/llap/orc_merge5.q.out
@@ -373,6 +373,8 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
+ Concatenate
+ table name: default.orc_merge5b_n0
Stage: Stage-1
Move Operator
diff --git a/ql/src/test/results/clientpositive/llap/orc_merge6.q.out b/ql/src/test/results/clientpositive/llap/orc_merge6.q.out
index 3968340..95cbe1e 100644
--- a/ql/src/test/results/clientpositive/llap/orc_merge6.q.out
+++ b/ql/src/test/results/clientpositive/llap/orc_merge6.q.out
@@ -521,6 +521,11 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
+ Concatenate
+ partition desc:
+ hour 24
+ year 2000
+ table name: default.orc_merge5a_n1
Stage: Stage-1
Move Operator
diff --git a/ql/src/test/results/clientpositive/llap/orc_merge7.q.out b/ql/src/test/results/clientpositive/llap/orc_merge7.q.out
index 8472cfb..4155dc6 100644
--- a/ql/src/test/results/clientpositive/llap/orc_merge7.q.out
+++ b/ql/src/test/results/clientpositive/llap/orc_merge7.q.out
@@ -652,6 +652,10 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
+ Concatenate
+ partition desc:
+ st 80.0
+ table name: default.orc_merge5a_n0
Stage: Stage-1
Move Operator
diff --git a/ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out b/ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out
index 319bac7..48217cd 100644
--- a/ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out
+++ b/ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out
@@ -337,6 +337,10 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
+ Concatenate
+ partition desc:
+ st 80.0
+ table name: default.orc_merge5a
Stage: Stage-1
Move Operator
diff --git a/ql/src/test/results/clientpositive/orc_merge10.q.out b/ql/src/test/results/clientpositive/orc_merge10.q.out
index b234381..e8ebd4e 100644
--- a/ql/src/test/results/clientpositive/orc_merge10.q.out
+++ b/ql/src/test/results/clientpositive/orc_merge10.q.out
@@ -602,6 +602,11 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
+ Concatenate
+ partition desc:
+ ds 1
+ part 0
+ table name: default.orcfile_merge1
Stage: Stage-1
Move Operator
diff --git a/ql/src/test/results/clientpositive/orc_merge5.q.out b/ql/src/test/results/clientpositive/orc_merge5.q.out
index 88d93de..df5651e 100644
--- a/ql/src/test/results/clientpositive/orc_merge5.q.out
+++ b/ql/src/test/results/clientpositive/orc_merge5.q.out
@@ -337,6 +337,8 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
+ Concatenate
+ table name: default.orc_merge5b_n0
Stage: Stage-1
Move Operator
diff --git a/ql/src/test/results/clientpositive/orc_merge6.q.out b/ql/src/test/results/clientpositive/orc_merge6.q.out
index bd01620..a07cfd3 100644
--- a/ql/src/test/results/clientpositive/orc_merge6.q.out
+++ b/ql/src/test/results/clientpositive/orc_merge6.q.out
@@ -485,6 +485,11 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
+ Concatenate
+ partition desc:
+ hour 24
+ year 2000
+ table name: default.orc_merge5a_n1
Stage: Stage-1
Move Operator
diff --git a/ql/src/test/results/clientpositive/orc_merge_incompat2.q.out b/ql/src/test/results/clientpositive/orc_merge_incompat2.q.out
index e5930b2..71ae287 100644
--- a/ql/src/test/results/clientpositive/orc_merge_incompat2.q.out
+++ b/ql/src/test/results/clientpositive/orc_merge_incompat2.q.out
@@ -332,6 +332,10 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
+ Concatenate
+ partition desc:
+ st 80.0
+ table name: default.orc_merge5a
Stage: Stage-1
Move Operator
diff --git a/ql/src/test/results/clientpositive/set_tblproperties.q.out b/ql/src/test/results/clientpositive/set_tblproperties.q.out
new file mode 100644
index 0000000..3503ea5
--- /dev/null
+++ b/ql/src/test/results/clientpositive/set_tblproperties.q.out
@@ -0,0 +1,102 @@
+PREHOOK: query: CREATE TABLE t (a INT)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t
+POSTHOOK: query: CREATE TABLE t (a INT)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t
+PREHOOK: query: EXPLAIN ALTER TABLE t SET TBLPROPERTIES('a'='x', 'b'='y')
+PREHOOK: type: ALTERTABLE_PROPERTIES
+PREHOOK: Input: default@t
+PREHOOK: Output: default@t
+POSTHOOK: query: EXPLAIN ALTER TABLE t SET TBLPROPERTIES('a'='x', 'b'='y')
+POSTHOOK: type: ALTERTABLE_PROPERTIES
+POSTHOOK: Input: default@t
+POSTHOOK: Output: default@t
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Set Properties
+ properties:
+ a x
+ b y
+ table name: default.t
+
+PREHOOK: query: ALTER TABLE t SET TBLPROPERTIES('a'='x', 'b'='y')
+PREHOOK: type: ALTERTABLE_PROPERTIES
+PREHOOK: Input: default@t
+PREHOOK: Output: default@t
+POSTHOOK: query: ALTER TABLE t SET TBLPROPERTIES('a'='x', 'b'='y')
+POSTHOOK: type: ALTERTABLE_PROPERTIES
+POSTHOOK: Input: default@t
+POSTHOOK: Output: default@t
+PREHOOK: query: SHOW CREATE TABLE t
+PREHOOK: type: SHOW_CREATETABLE
+PREHOOK: Input: default@t
+POSTHOOK: query: SHOW CREATE TABLE t
+POSTHOOK: type: SHOW_CREATETABLE
+POSTHOOK: Input: default@t
+CREATE TABLE `t`(
+ `a` int)
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+STORED AS INPUTFORMAT
+ 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
+LOCATION
+#### A masked pattern was here ####
+TBLPROPERTIES (
+ 'a'='x',
+ 'b'='y',
+ 'bucketing_version'='2',
+#### A masked pattern was here ####
+PREHOOK: query: EXPLAIN ALTER TABLE t UNSET TBLPROPERTIES('a', 'b')
+PREHOOK: type: ALTERTABLE_PROPERTIES
+PREHOOK: Input: default@t
+PREHOOK: Output: default@t
+POSTHOOK: query: EXPLAIN ALTER TABLE t UNSET TBLPROPERTIES('a', 'b')
+POSTHOOK: type: ALTERTABLE_PROPERTIES
+POSTHOOK: Input: default@t
+POSTHOOK: Output: default@t
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Unset Properties
+ properties:
+ a
+ b
+ table name: default.t
+
+PREHOOK: query: ALTER TABLE t UNSET TBLPROPERTIES('a', 'b')
+PREHOOK: type: ALTERTABLE_PROPERTIES
+PREHOOK: Input: default@t
+PREHOOK: Output: default@t
+POSTHOOK: query: ALTER TABLE t UNSET TBLPROPERTIES('a', 'b')
+POSTHOOK: type: ALTERTABLE_PROPERTIES
+POSTHOOK: Input: default@t
+POSTHOOK: Output: default@t
+PREHOOK: query: SHOW CREATE TABLE t
+PREHOOK: type: SHOW_CREATETABLE
+PREHOOK: Input: default@t
+POSTHOOK: query: SHOW CREATE TABLE t
+POSTHOOK: type: SHOW_CREATETABLE
+POSTHOOK: Input: default@t
+CREATE TABLE `t`(
+ `a` int)
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+STORED AS INPUTFORMAT
+ 'org.apache.hadoop.mapred.TextInputFormat'
+OUTPUTFORMAT
+ 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
+LOCATION
+#### A masked pattern was here ####
+TBLPROPERTIES (
+ 'bucketing_version'='2',
+#### A masked pattern was here ####
diff --git a/ql/src/test/results/clientpositive/spark/orc_merge5.q.out b/ql/src/test/results/clientpositive/spark/orc_merge5.q.out
index fe5a71e..e72d1d2 100644
--- a/ql/src/test/results/clientpositive/spark/orc_merge5.q.out
+++ b/ql/src/test/results/clientpositive/spark/orc_merge5.q.out
@@ -289,6 +289,8 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
+ Concatenate
+ table name: default.orc_merge5b_n0
Stage: Stage-1
Move Operator
diff --git a/ql/src/test/results/clientpositive/spark/orc_merge6.q.out b/ql/src/test/results/clientpositive/spark/orc_merge6.q.out
index ccf766d..982b614 100644
--- a/ql/src/test/results/clientpositive/spark/orc_merge6.q.out
+++ b/ql/src/test/results/clientpositive/spark/orc_merge6.q.out
@@ -421,6 +421,11 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
+ Concatenate
+ partition desc:
+ hour 24
+ year 2000
+ table name: default.orc_merge5a_n1
Stage: Stage-1
Move Operator
diff --git a/ql/src/test/results/clientpositive/spark/orc_merge7.q.out b/ql/src/test/results/clientpositive/spark/orc_merge7.q.out
index 192f8c4..a641ed7 100644
--- a/ql/src/test/results/clientpositive/spark/orc_merge7.q.out
+++ b/ql/src/test/results/clientpositive/spark/orc_merge7.q.out
@@ -552,6 +552,10 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
+ Concatenate
+ partition desc:
+ st 80.0
+ table name: default.orc_merge5a_n0
Stage: Stage-1
Move Operator
diff --git a/ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out b/ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out
index 2330d9e..1bfbf4d 100644
--- a/ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out
+++ b/ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out
@@ -293,6 +293,10 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
+ Concatenate
+ partition desc:
+ st 80.0
+ table name: default.orc_merge5a
Stage: Stage-1
Move Operator
diff --git a/ql/src/test/results/clientpositive/table_set_owner.q.out b/ql/src/test/results/clientpositive/table_set_owner.q.out
new file mode 100644
index 0000000..8528faf
--- /dev/null
+++ b/ql/src/test/results/clientpositive/table_set_owner.q.out
@@ -0,0 +1,28 @@
+PREHOOK: query: CREATE TABLE t (a INT)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t
+POSTHOOK: query: CREATE TABLE t (a INT)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t
+PREHOOK: query: EXPLAIN ALTER TABLE t SET OWNER USER user1
+PREHOOK: type: ALTERTABLE_OWNER
+POSTHOOK: query: EXPLAIN ALTER TABLE t SET OWNER USER user1
+POSTHOOK: type: ALTERTABLE_OWNER
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+#### A masked pattern was here ####
+ Principal
+ name: user1
+ table name: default.t
+
+PREHOOK: query: ALTER TABLE t SET OWNER USER user1
+PREHOOK: type: ALTERTABLE_OWNER
+POSTHOOK: query: ALTER TABLE t SET OWNER USER user1
+POSTHOOK: type: ALTERTABLE_OWNER
+POSTHOOK: Input: default@t
+POSTHOOK: Output: default@t
diff --git a/ql/src/test/results/clientpositive/table_storage.q.out b/ql/src/test/results/clientpositive/table_storage.q.out
index 4311350..7de66e6 100644
--- a/ql/src/test/results/clientpositive/table_storage.q.out
+++ b/ql/src/test/results/clientpositive/table_storage.q.out
@@ -509,7 +509,7 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
Set Serde Props
- props:
+ properties:
property1 value1
property2 value2
table name: default.t
diff --git a/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out b/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out
index adde248..82666ad 100644
--- a/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out
+++ b/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out
@@ -747,6 +747,23 @@ POSTHOOK: query: select ROW__ID, count(*) from over10k_orc_bucketed_n0 group by
POSTHOOK: type: QUERY
POSTHOOK: Input: default@over10k_orc_bucketed_n0
POSTHOOK: Output: hdfs://### HDFS PATH ###
+PREHOOK: query: explain alter table over10k_orc_bucketed_n0 compact 'major' WITH OVERWRITE TBLPROPERTIES ('compactor.mapreduce.map.memory.mb'='500', 'compactor.mapreduce.reduce.memory.mb'='500','compactor.mapreduce.map.memory.mb'='500', 'compactor.hive.tez.container.size'='500')
+PREHOOK: type: ALTERTABLE_COMPACT
+POSTHOOK: query: explain alter table over10k_orc_bucketed_n0 compact 'major' WITH OVERWRITE TBLPROPERTIES ('compactor.mapreduce.map.memory.mb'='500', 'compactor.mapreduce.reduce.memory.mb'='500','compactor.mapreduce.map.memory.mb'='500', 'compactor.hive.tez.container.size'='500')
+POSTHOOK: type: ALTERTABLE_COMPACT
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Compact
+ compaction type: major
+ properties:
+ compactor.hive.tez.container.size 500
+ compactor.mapreduce.map.memory.mb 500
+ compactor.mapreduce.reduce.memory.mb 500
+ table name: default.over10k_orc_bucketed_n0
+
PREHOOK: query: alter table over10k_orc_bucketed_n0 compact 'major' WITH OVERWRITE TBLPROPERTIES ('compactor.mapreduce.map.memory.mb'='500', 'compactor.mapreduce.reduce.memory.mb'='500','compactor.mapreduce.map.memory.mb'='500', 'compactor.hive.tez.container.size'='500')
PREHOOK: type: ALTERTABLE_COMPACT
POSTHOOK: query: alter table over10k_orc_bucketed_n0 compact 'major' WITH OVERWRITE TBLPROPERTIES ('compactor.mapreduce.map.memory.mb'='500', 'compactor.mapreduce.reduce.memory.mb'='500','compactor.mapreduce.map.memory.mb'='500', 'compactor.hive.tez.container.size'='500')
diff --git a/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out b/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
index 1ad26b5..6925f58 100644
--- a/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out
@@ -174,8 +174,7 @@ POSTHOOK: type: ALTERTABLE_RENAME
POSTHOOK: Input: newdb@tab_n2
POSTHOOK: Output: newdb@tab_n2
Stage-0
- Alter Table Operator:
- new name:newDB.newName,old name:newDB.tab_n2,type:rename
+ Rename Table{"new table name:":"newDB.newName","table name:":"newDB.tab_n2"}
PREHOOK: query: drop table tab_n2
PREHOOK: type: DROPTABLE
diff --git a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
index c07c6a3..26eae7e 100644
--- a/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
+++ b/ql/src/test/results/clientpositive/tez/explainuser_3.q.out
@@ -178,8 +178,7 @@ POSTHOOK: type: ALTERTABLE_RENAME
POSTHOOK: Input: newdb@tab_n1
POSTHOOK: Output: newdb@tab_n1
Stage-0
- Alter Table Operator:
- new name:newDB.newName,old name:newDB.tab_n1,type:rename
+ Rename Table{"new table name:":"newDB.newName","table name:":"newDB.tab_n1"}
PREHOOK: query: explain drop table tab_n1
PREHOOK: type: DROPTABLE
diff --git a/ql/src/test/results/clientpositive/touch.q.out b/ql/src/test/results/clientpositive/touch.q.out
index c239715..c58f600 100644
--- a/ql/src/test/results/clientpositive/touch.q.out
+++ b/ql/src/test/results/clientpositive/touch.q.out
@@ -44,6 +44,22 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
POSTHOOK: Output: tc@tstsrcpart@ds=2008-04-08/hr=12
POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: EXPLAIN ALTER TABLE tc.tstsrc TOUCH
+PREHOOK: type: ALTERTABLE_TOUCH
+PREHOOK: Input: tc@tstsrc
+PREHOOK: Output: tc@tstsrc
+POSTHOOK: query: EXPLAIN ALTER TABLE tc.tstsrc TOUCH
+POSTHOOK: type: ALTERTABLE_TOUCH
+POSTHOOK: Input: tc@tstsrc
+POSTHOOK: Output: tc@tstsrc
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Touch
+ table name: tc.tstsrc
+
PREHOOK: query: ALTER TABLE tc.tstsrc TOUCH
PREHOOK: type: ALTERTABLE_TOUCH
PREHOOK: Input: tc@tstsrc
@@ -60,6 +76,25 @@ POSTHOOK: query: ALTER TABLE tc.tstsrcpart TOUCH
POSTHOOK: type: ALTERTABLE_TOUCH
POSTHOOK: Input: tc@tstsrcpart
POSTHOOK: Output: tc@tstsrcpart
+PREHOOK: query: EXPLAIN ALTER TABLE tc.tstsrcpart TOUCH PARTITION (ds='2008-04-08', hr='12')
+PREHOOK: type: ALTERTABLE_TOUCH
+PREHOOK: Input: tc@tstsrcpart
+PREHOOK: Output: tc@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: EXPLAIN ALTER TABLE tc.tstsrcpart TOUCH PARTITION (ds='2008-04-08', hr='12')
+POSTHOOK: type: ALTERTABLE_TOUCH
+POSTHOOK: Input: tc@tstsrcpart
+POSTHOOK: Output: tc@tstsrcpart@ds=2008-04-08/hr=12
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Touch
+ partition spec:
+ ds 2008-04-08
+ hr 12
+ table name: tc.tstsrcpart
+
PREHOOK: query: ALTER TABLE tc.tstsrcpart TOUCH PARTITION (ds='2008-04-08', hr='12')
PREHOOK: type: ALTERTABLE_TOUCH
PREHOOK: Input: tc@tstsrcpart
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
index e33f5e4..57f4263 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
@@ -44,9 +44,9 @@ public interface HiveMetaHook {
public String ALTER_TABLE_OPERATION_TYPE = "alterTableOpType";
- // These should remain in sync with AlterTableDesc::AlterTableType enum
+ // These should remain in sync with AlterTableType enum
public List<String> allowedAlterTypes = ImmutableList.of("ADDPROPS", "DROPPROPS");
- String SET_LOCATION = "SET_LOCATION";
+ String ALTERLOCATION = "ALTERLOCATION";
/**
* Called before a new table definition is added to the metastore
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index 1249058..9756425 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -63,7 +63,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import static org.apache.hadoop.hive.metastore.HiveMetaHook.SET_LOCATION;
+import static org.apache.hadoop.hive.metastore.HiveMetaHook.ALTERLOCATION;
import static org.apache.hadoop.hive.metastore.HiveMetaHook.ALTER_TABLE_OPERATION_TYPE;
import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
@@ -825,7 +825,7 @@ public class HiveAlterHandler implements AlterHandler {
// new files list. So, it may cause data inconsistency.
if ((ec != null) && ec.isSetProperties()) {
String alterType = ec.getProperties().get(ALTER_TABLE_OPERATION_TYPE);
- if (alterType != null && alterType.equalsIgnoreCase(SET_LOCATION) &&
+ if (alterType != null && alterType.equalsIgnoreCase(ALTERLOCATION) &&
tbl.getTableType().equalsIgnoreCase(TableType.MANAGED_TABLE.name())) {
String tableName = TableName.getQualified(tbl.getCatName(), tbl.getDbName(), tbl.getTableName());
throw new InvalidOperationException(
@@ -848,7 +848,7 @@ public class HiveAlterHandler implements AlterHandler {
// managed is true on the source cluster.
if ((ec != null) && ec.isSetProperties()) {
String alterType = ec.getProperties().get(ALTER_TABLE_OPERATION_TYPE);
- if (alterType != null && alterType.equalsIgnoreCase(SET_LOCATION) &&
+ if (alterType != null && alterType.equalsIgnoreCase(ALTERLOCATION) &&
oldTbl.getTableType().equalsIgnoreCase(TableType.MANAGED_TABLE.name())) {
String tableName = TableName.getQualified(oldTbl.getCatName(), oldTbl.getDbName(), oldTbl.getTableName());
throw new InvalidOperationException(