You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2019/07/05 16:13:57 UTC
[hive] branch master updated: HIVE-21947: Move Materialized View
Update under DDL (Miklos Gergely, reviewed by Jesus Camacho Rodriguez)
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 6e4c500 HIVE-21947: Move Materialized View Update under DDL (Miklos Gergely, reviewed by Jesus Camacho Rodriguez)
6e4c500 is described below
commit 6e4c5006c2834e4c7a671db834b43157a182812c
Author: miklosgergely <mg...@cloudera.com>
AuthorDate: Wed Jul 3 01:18:17 2019 +0200
HIVE-21947: Move Materialized View Update under DDL (Miklos Gergely, reviewed by Jesus Camacho Rodriguez)
Close apache/hive#697
---
.../ql/ddl/view/AlterMaterializedViewDesc.java | 51 ------------
.../ddl/view/AlterMaterializedViewRewriteDesc.java | 18 +++--
.../view/MaterializedViewUpdateDesc.java} | 29 ++++---
.../ddl/view/MaterializedViewUpdateOperation.java | 71 +++++++++++++++++
.../hadoop/hive/ql/exec/MaterializedViewTask.java | 91 ----------------------
.../apache/hadoop/hive/ql/exec/TaskFactory.java | 3 -
.../apache/hadoop/hive/ql/parse/ParseContext.java | 9 ++-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 6 +-
.../apache/hadoop/hive/ql/parse/TaskCompiler.java | 34 ++++----
.../materialized_view_create_rewrite_dummy.q | 3 +
.../test/results/clientnegative/masking_mv.q.out | 4 +-
.../beeline/materialized_view_create_rewrite.q.out | 4 +-
.../clientpositive/druid/druidmini_mv.q.out | 4 +-
.../llap/materialized_view_create_rewrite.q.out | 4 +-
.../llap/materialized_view_create_rewrite_3.q.out | 8 +-
.../llap/materialized_view_create_rewrite_4.q.out | 23 ++++--
.../llap/materialized_view_create_rewrite_5.q.out | 16 +++-
.../materialized_view_create_rewrite_dummy.q.out | 23 +++++-
...ialized_view_create_rewrite_rebuild_dummy.q.out | 8 +-
...erialized_view_create_rewrite_time_window.q.out | 11 ++-
.../llap/materialized_view_partitioned.q.out | 12 ++-
.../llap/materialized_view_partitioned_3.q.out | 4 +-
.../test/results/clientpositive/masking_mv.q.out | 8 +-
23 files changed, 230 insertions(+), 214 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/AlterMaterializedViewDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/AlterMaterializedViewDesc.java
deleted file mode 100644
index 7f8b16b..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/AlterMaterializedViewDesc.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.ddl.view;
-
-import java.io.Serializable;
-
-import org.apache.hadoop.hive.ql.plan.Explain;
-import org.apache.hadoop.hive.ql.ddl.DDLDesc;
-import org.apache.hadoop.hive.ql.plan.Explain.Level;
-
-/**
- * DDL task description for all the ALTER MATERIALIZED VIEW commands.
- */
-@Explain(displayName = "Alter Materialized View", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
-abstract class AlterMaterializedViewDesc implements DDLDesc, Serializable {
- private static final long serialVersionUID = 1L;
-
- /**
- * Alter Materialized View Types.
- */
- enum AlterMaterializedViewTypes {
- UPDATE_REWRITE_FLAG
- };
-
- private AlterMaterializedViewTypes op;
-
- AlterMaterializedViewDesc(AlterMaterializedViewTypes type) {
- this.op = type;
- }
-
- @Explain(displayName = "operation", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
- public String getOpString() {
- return op.toString();
- }
-}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/AlterMaterializedViewRewriteDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/AlterMaterializedViewRewriteDesc.java
index bcf0977..0e3df61 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/AlterMaterializedViewRewriteDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/AlterMaterializedViewRewriteDesc.java
@@ -23,17 +23,14 @@ import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
/**
- * DDL task description for the ALTER MATERIALIZED VIEW commands.
+ * DDL task description for the ALTER MATERIALIZED VIEW (ENABLE|DISABLE) REWRITE commands.
*/
-@Explain(displayName = "Alter Materialized View", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
-public class AlterMaterializedViewRewriteDesc extends AlterMaterializedViewDesc implements DDLDescWithWriteId {
- private static final long serialVersionUID = 1L;
-
+@Explain(displayName = "Alter Materialized View Rewrite", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+public class AlterMaterializedViewRewriteDesc implements DDLDescWithWriteId {
private final String fqMaterializedViewName;
private final boolean rewriteEnable;
public AlterMaterializedViewRewriteDesc(String fqMaterializedViewName, boolean rewriteEnable) {
- super(AlterMaterializedViewTypes.UPDATE_REWRITE_FLAG);
this.fqMaterializedViewName = fqMaterializedViewName;
this.rewriteEnable = rewriteEnable;
}
@@ -43,10 +40,19 @@ public class AlterMaterializedViewRewriteDesc extends AlterMaterializedViewDesc
return fqMaterializedViewName;
}
+ @Explain(displayName = "enable", displayOnlyOnTrue = true,
+ explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public boolean isRewriteEnable() {
return rewriteEnable;
}
+ /** Only for explaining. */
+ @Explain(displayName = "disable", displayOnlyOnTrue = true,
+ explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+ public boolean isRewriteDisable() {
+ return !rewriteEnable;
+ }
+
@Override
public void setWriteId(long writeId) {
// We don't actually need the write id, but by implementing DDLDescWithWriteId it ensures that it is allocated
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/MaterializedViewUpdateDesc.java
similarity index 59%
rename from ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewDesc.java
rename to ql/src/java/org/apache/hadoop/hive/ql/ddl/view/MaterializedViewUpdateDesc.java
index f7af073..674a256 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/MaterializedViewUpdateDesc.java
@@ -15,42 +15,53 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hive.ql.exec;
+package org.apache.hadoop.hive.ql.ddl.view;
-import org.apache.hadoop.hive.metastore.api.CreationMetadata;
+import org.apache.hadoop.hive.ql.ddl.DDLDesc;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
import java.io.Serializable;
-@Explain(displayName = "Materialized View Work", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
-public class MaterializedViewDesc implements Serializable {
+/**
+ * DDL task description of updating a materialized view.
+ */
+@Explain(displayName = "Materialized View Update", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+public class MaterializedViewUpdateDesc implements DDLDesc, Serializable {
private static final long serialVersionUID = 1L;
- private final String viewName;
+
+ private final String name;
private final boolean retrieveAndInclude;
private final boolean disableRewrite;
private final boolean updateCreationMetadata;
- public MaterializedViewDesc(String viewName, boolean retrieveAndInclude, boolean disableRewrite,
+ public MaterializedViewUpdateDesc(String name, boolean retrieveAndInclude, boolean disableRewrite,
boolean updateCreationMetadata) {
- this.viewName = viewName;
+ this.name = name;
this.retrieveAndInclude = retrieveAndInclude;
this.disableRewrite = disableRewrite;
this.updateCreationMetadata = updateCreationMetadata;
}
- public String getViewName() {
- return viewName;
+ @Explain(displayName = "name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+ public String getName() {
+ return name;
}
+ @Explain(displayName = "retrieveAndInclude", displayOnlyOnTrue = true,
+ explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public boolean isRetrieveAndInclude() {
return retrieveAndInclude;
}
+ @Explain(displayName = "disableRewrite", displayOnlyOnTrue = true,
+ explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public boolean isDisableRewrite() {
return disableRewrite;
}
+ @Explain(displayName = "updateCreationMetadata", displayOnlyOnTrue = true,
+ explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public boolean isUpdateCreationMetadata() {
return updateCreationMetadata;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/MaterializedViewUpdateOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/MaterializedViewUpdateOperation.java
new file mode 100644
index 0000000..ad6e163
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/MaterializedViewUpdateOperation.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.view;
+
+import org.apache.hadoop.hive.common.ValidTxnWriteIdList;
+import org.apache.hadoop.hive.metastore.api.CreationMetadata;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
+import org.apache.hadoop.hive.ql.ddl.DDLOperation;
+import org.apache.hadoop.hive.ql.ddl.DDLOperationContext;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.HiveMaterializedViewsRegistry;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState;
+
+import com.google.common.collect.ImmutableSet;
+
+/**
+ * Operation process of updating a materialized view.
+ */
+public class MaterializedViewUpdateOperation extends DDLOperation<MaterializedViewUpdateDesc> {
+ public MaterializedViewUpdateOperation(DDLOperationContext context, MaterializedViewUpdateDesc desc) {
+ super(context, desc);
+ }
+
+ @Override
+ public int execute() throws HiveException {
+ if (context.getDriverContext().getCtx().getExplainAnalyze() == AnalyzeState.RUNNING) {
+ return 0;
+ }
+
+ try {
+ if (desc.isRetrieveAndInclude()) {
+ Table mvTable = context.getDb().getTable(desc.getName());
+ HiveMaterializedViewsRegistry.get().createMaterializedView(context.getDb().getConf(), mvTable);
+ } else if (desc.isDisableRewrite()) {
+ // Disabling rewriting, removing from cache
+ String[] names = desc.getName().split("\\.");
+ HiveMaterializedViewsRegistry.get().dropMaterializedView(names[0], names[1]);
+ } else if (desc.isUpdateCreationMetadata()) {
+ // We need to update the status of the creation signature
+ Table mvTable = context.getDb().getTable(desc.getName());
+ CreationMetadata cm = new CreationMetadata(MetaStoreUtils.getDefaultCatalog(context.getConf()),
+ mvTable.getDbName(), mvTable.getTableName(),
+ ImmutableSet.copyOf(mvTable.getCreationMetadata().getTablesUsed()));
+ cm.setValidTxnList(context.getConf().get(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY));
+ context.getDb().updateCreationMetadata(mvTable.getDbName(), mvTable.getTableName(), cm);
+ }
+ } catch (HiveException e) {
+ LOG.debug("Exception during materialized view cache update", e);
+ context.getTask().setException(e);
+ }
+
+ return 0;
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewTask.java
deleted file mode 100644
index 87828b1..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MaterializedViewTask.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec;
-
-import com.google.common.collect.ImmutableSet;
-import org.apache.hadoop.hive.common.ValidTxnWriteIdList;
-import org.apache.hadoop.hive.metastore.api.CreationMetadata;
-import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
-import org.apache.hadoop.hive.ql.DriverContext;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.HiveMaterializedViewsRegistry;
-import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState;
-import org.apache.hadoop.hive.ql.plan.api.StageType;
-
-import java.io.Serializable;
-
-import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
-
-/**
- * This task does some work related to materialized views. In particular, it adds
- * or removes the materialized view from the registry if needed, or registers new
- * creation metadata.
- */
-public class MaterializedViewTask extends Task<MaterializedViewDesc> implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- public MaterializedViewTask() {
- super();
- }
-
- @Override
- public int execute(DriverContext driverContext) {
- if (driverContext.getCtx().getExplainAnalyze() == AnalyzeState.RUNNING) {
- return 0;
- }
- try {
- if (getWork().isRetrieveAndInclude()) {
- Hive db = Hive.get(conf);
- Table mvTable = db.getTable(getWork().getViewName());
- HiveMaterializedViewsRegistry.get().createMaterializedView(db.getConf(), mvTable);
- } else if (getWork().isDisableRewrite()) {
- // Disabling rewriting, removing from cache
- String[] names = getWork().getViewName().split("\\.");
- HiveMaterializedViewsRegistry.get().dropMaterializedView(names[0], names[1]);
- } else if (getWork().isUpdateCreationMetadata()) {
- // We need to update the status of the creation signature
- Hive db = Hive.get(conf);
- Table mvTable = db.getTable(getWork().getViewName());
- CreationMetadata cm =
- new CreationMetadata(MetaStoreUtils.getDefaultCatalog(conf), mvTable.getDbName(),
- mvTable.getTableName(),
- ImmutableSet.copyOf(mvTable.getCreationMetadata().getTablesUsed()));
- cm.setValidTxnList(conf.get(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY));
- db.updateCreationMetadata(mvTable.getDbName(), mvTable.getTableName(), cm);
- }
- } catch (HiveException e) {
- LOG.debug("Exception during materialized view cache update", e);
- setException(e);
- }
- return 0;
- }
-
- @Override
- public StageType getType() {
- return StageType.DDL;
- }
-
- @Override
- public String getName() {
- return MaterializedViewTask.class.getSimpleName();
- }
-}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
index 7025b4a..f707264 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
@@ -88,9 +88,6 @@ public final class TaskFactory {
taskvec.add(new TaskTuple<CopyWork>(CopyWork.class, CopyTask.class));
taskvec.add(new TaskTuple<ReplCopyWork>(ReplCopyWork.class, ReplCopyTask.class));
taskvec.add(new TaskTuple<DDLWork>(DDLWork.class, DDLTask.class));
- taskvec.add(new TaskTuple<MaterializedViewDesc>(
- MaterializedViewDesc.class,
- MaterializedViewTask.class));
taskvec
.add(new TaskTuple<ExplainWork>(ExplainWork.class, ExplainTask.class));
taskvec
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
index 1c6d4ac..67b4901 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
@@ -25,13 +25,13 @@ import org.apache.hadoop.hive.ql.QueryProperties;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.ddl.table.creation.CreateTableDesc;
import org.apache.hadoop.hive.ql.ddl.view.CreateViewDesc;
+import org.apache.hadoop.hive.ql.ddl.view.MaterializedViewUpdateDesc;
import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator;
import org.apache.hadoop.hive.ql.exec.FetchTask;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
import org.apache.hadoop.hive.ql.exec.JoinOperator;
import org.apache.hadoop.hive.ql.exec.ListSinkOperator;
import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
-import org.apache.hadoop.hive.ql.exec.MaterializedViewDesc;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator;
@@ -122,7 +122,7 @@ public class ParseContext {
private AnalyzeRewriteContext analyzeRewrite;
private CreateTableDesc createTableDesc;
private CreateViewDesc createViewDesc;
- private MaterializedViewDesc materializedViewUpdateDesc;
+ private MaterializedViewUpdateDesc materializedViewUpdateDesc;
private boolean reduceSinkAddedBySortedDynPartition;
private Map<SelectOperator, Table> viewProjectToViewSchema;
@@ -199,7 +199,8 @@ public class ParseContext {
Map<String, ReadEntity> viewAliasToInput,
List<ReduceSinkOperator> reduceSinkOperatorsAddedByEnforceBucketingSorting,
AnalyzeRewriteContext analyzeRewrite, CreateTableDesc createTableDesc,
- CreateViewDesc createViewDesc, MaterializedViewDesc materializedViewUpdateDesc, QueryProperties queryProperties,
+ CreateViewDesc createViewDesc, MaterializedViewUpdateDesc materializedViewUpdateDesc,
+ QueryProperties queryProperties,
Map<SelectOperator, Table> viewProjectToTableSchema, Set<FileSinkDesc> acidFileSinks) {
this.queryState = queryState;
this.conf = queryState.getConf();
@@ -611,7 +612,7 @@ public class ParseContext {
return createViewDesc;
}
- public MaterializedViewDesc getMaterializedViewUpdateDesc() {
+ public MaterializedViewUpdateDesc getMaterializedViewUpdateDesc() {
return materializedViewUpdateDesc;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index e09fc37..7c58072 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -107,6 +107,7 @@ import org.apache.hadoop.hive.ql.ddl.table.creation.CreateTableLikeDesc;
import org.apache.hadoop.hive.ql.ddl.table.misc.AlterTableUnsetPropertiesDesc;
import org.apache.hadoop.hive.ql.ddl.table.misc.PreInsertTableDesc;
import org.apache.hadoop.hive.ql.ddl.view.CreateViewDesc;
+import org.apache.hadoop.hive.ql.ddl.view.MaterializedViewUpdateDesc;
import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator;
import org.apache.hadoop.hive.ql.exec.ArchiveUtils;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
@@ -120,7 +121,6 @@ import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
import org.apache.hadoop.hive.ql.exec.JoinOperator;
import org.apache.hadoop.hive.ql.exec.LimitOperator;
-import org.apache.hadoop.hive.ql.exec.MaterializedViewDesc;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.OperatorFactory;
import org.apache.hadoop.hive.ql.exec.RecordReader;
@@ -338,7 +338,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
Map<String, PrunedPartitionList> prunedPartitions;
protected List<FieldSchema> resultSchema;
protected CreateViewDesc createVwDesc;
- protected MaterializedViewDesc materializedViewUpdateDesc;
+ protected MaterializedViewUpdateDesc materializedViewUpdateDesc;
protected ArrayList<String> viewsExpanded;
protected ASTNode viewSelect;
protected final UnparseTranslator unparseTranslator;
@@ -7362,7 +7362,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
if (destinationTable.isMaterializedView()) {
- materializedViewUpdateDesc = new MaterializedViewDesc(
+ materializedViewUpdateDesc = new MaterializedViewUpdateDesc(
destinationTable.getFullyQualifiedName(), false, false, true);
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
index 0a7fa5a..0b55a39 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
@@ -38,9 +38,9 @@ import org.apache.hadoop.hive.ql.ddl.DDLWork;
import org.apache.hadoop.hive.ql.ddl.table.creation.CreateTableDesc;
import org.apache.hadoop.hive.ql.ddl.view.AlterMaterializedViewRewriteDesc;
import org.apache.hadoop.hive.ql.ddl.view.CreateViewDesc;
+import org.apache.hadoop.hive.ql.ddl.view.MaterializedViewUpdateDesc;
import org.apache.hadoop.hive.ql.exec.FetchTask;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
-import org.apache.hadoop.hive.ql.exec.MaterializedViewDesc;
import org.apache.hadoop.hive.ql.exec.MoveTask;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.OperatorUtils;
@@ -361,20 +361,23 @@ public abstract class TaskCompiler {
CreateTableDesc crtTblDesc = pCtx.getCreateTable();
crtTblDesc.validate(conf);
Task<? extends Serializable> crtTblTask = TaskFactory.get(new DDLWork(inputs, outputs, crtTblDesc));
- patchUpAfterCTASorMaterializedView(rootTasks, outputs, crtTblTask, CollectionUtils.isEmpty(crtTblDesc.getPartColNames()));
+ patchUpAfterCTASorMaterializedView(rootTasks, inputs, outputs, crtTblTask,
+ CollectionUtils.isEmpty(crtTblDesc.getPartColNames()));
} else if (pCtx.getQueryProperties().isMaterializedView()) {
// generate a DDL task and make it a dependent task of the leaf
CreateViewDesc viewDesc = pCtx.getCreateViewDesc();
Task<? extends Serializable> crtViewTask = TaskFactory.get(new DDLWork(
inputs, outputs, viewDesc));
- patchUpAfterCTASorMaterializedView(rootTasks, outputs, crtViewTask, CollectionUtils.isEmpty(viewDesc.getPartColNames()));
+ patchUpAfterCTASorMaterializedView(rootTasks, inputs, outputs, crtViewTask,
+ CollectionUtils.isEmpty(viewDesc.getPartColNames()));
} else if (pCtx.getMaterializedViewUpdateDesc() != null) {
// If there is a materialized view update desc, we create introduce it at the end
// of the tree.
- MaterializedViewDesc materializedViewDesc = pCtx.getMaterializedViewUpdateDesc();
+ MaterializedViewUpdateDesc materializedViewDesc = pCtx.getMaterializedViewUpdateDesc();
+ DDLWork ddlWork = new DDLWork(inputs, outputs, materializedViewDesc);
Set<Task<? extends Serializable>> leafTasks = new LinkedHashSet<Task<? extends Serializable>>();
getLeafTasks(rootTasks, leafTasks);
- Task<? extends Serializable> materializedViewTask = TaskFactory.get(materializedViewDesc, conf);
+ Task<? extends Serializable> materializedViewTask = TaskFactory.get(ddlWork, conf);
for (Task<? extends Serializable> task : leafTasks) {
task.addDependentTask(materializedViewTask);
}
@@ -491,10 +494,9 @@ public abstract class TaskCompiler {
}
}
- private void patchUpAfterCTASorMaterializedView(final List<Task<? extends Serializable>> rootTasks,
- final HashSet<WriteEntity> outputs,
- Task<? extends Serializable> createTask,
- boolean createTaskAfterMoveTask) {
+ private void patchUpAfterCTASorMaterializedView(List<Task<? extends Serializable>> rootTasks,
+ Set<ReadEntity> inputs, Set<WriteEntity> outputs, Task<? extends Serializable> createTask,
+ boolean createTaskAfterMoveTask) {
// clear the mapredWork output file from outputs for CTAS
// DDLWork at the tail of the chain will have the output
Iterator<WriteEntity> outIter = outputs.iterator();
@@ -510,7 +512,7 @@ public abstract class TaskCompiler {
}
// find all leaf tasks and make the DDLTask as a dependent task on all of them
- HashSet<Task<? extends Serializable>> leaves = new LinkedHashSet<>();
+ Set<Task<? extends Serializable>> leaves = new LinkedHashSet<>();
getLeafTasks(rootTasks, leaves);
assert (leaves.size() > 0);
// Target task is supposed to be the last task
@@ -554,16 +556,20 @@ public abstract class TaskCompiler {
if (createViewDesc.isMaterialized()) {
String tableName = createViewDesc.getViewName();
boolean retrieveAndInclude = createViewDesc.isRewriteEnabled();
- targetTask.addDependentTask(TaskFactory.get(
- new MaterializedViewDesc(tableName, retrieveAndInclude, false, false), conf));
+ MaterializedViewUpdateDesc materializedViewUpdateDesc =
+ new MaterializedViewUpdateDesc(tableName, retrieveAndInclude, false, false);
+ DDLWork ddlWork = new DDLWork(inputs, outputs, materializedViewUpdateDesc);
+ targetTask.addDependentTask(TaskFactory.get(ddlWork, conf));
}
} else if (desc instanceof AlterMaterializedViewRewriteDesc) {
AlterMaterializedViewRewriteDesc alterMVRewriteDesc = (AlterMaterializedViewRewriteDesc)desc;
String tableName = alterMVRewriteDesc.getMaterializedViewName();
boolean retrieveAndInclude = alterMVRewriteDesc.isRewriteEnable();
boolean disableRewrite = !alterMVRewriteDesc.isRewriteEnable();
- targetTask.addDependentTask(
- TaskFactory.get(new MaterializedViewDesc(tableName, retrieveAndInclude, disableRewrite, false), conf));
+ MaterializedViewUpdateDesc materializedViewUpdateDesc =
+ new MaterializedViewUpdateDesc(tableName, retrieveAndInclude, disableRewrite, false);
+ DDLWork ddlWork = new DDLWork(inputs, outputs, materializedViewUpdateDesc);
+ targetTask.addDependentTask(TaskFactory.get(ddlWork, conf));
}
}
}
diff --git a/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_dummy.q b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_dummy.q
index 6978593..82c4fb9 100644
--- a/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_dummy.q
+++ b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_dummy.q
@@ -35,6 +35,9 @@ select a, c from cmv_basetable_n0 where a = 3;
select a, c from cmv_basetable_n0 where a = 3;
+explain
+alter materialized view cmv_mat_view2 disable rewrite;
+
alter materialized view cmv_mat_view2 disable rewrite;
explain
diff --git a/ql/src/test/results/clientnegative/masking_mv.q.out b/ql/src/test/results/clientnegative/masking_mv.q.out
index 1e18b36..eb19678 100644
--- a/ql/src/test/results/clientnegative/masking_mv.q.out
+++ b/ql/src/test/results/clientnegative/masking_mv.q.out
@@ -117,7 +117,9 @@ STAGE PLANS:
Table: default.masking_test_view_n_mv
Stage: Stage-9
- Materialized View Work
+ Materialized View Update
+ name: default.masking_test_view_n_mv
+ retrieveAndInclude: true
Stage: Stage-3
Merge File Operator
diff --git a/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out b/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out
index e9753db..728ae97 100644
--- a/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out
+++ b/ql/src/test/results/clientpositive/beeline/materialized_view_create_rewrite.q.out
@@ -263,9 +263,9 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
- Alter Materialized View
+ Alter Materialized View Rewrite
name: default.cmv_mat_view2_n4
- operation: UPDATE_REWRITE_FLAG
+ enable: true
PREHOOK: query: alter materialized view cmv_mat_view2_n4 enable rewrite
PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE
diff --git a/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out b/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out
index 96374bb..281113b 100644
--- a/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out
+++ b/ql/src/test/results/clientpositive/druid/druidmini_mv.q.out
@@ -477,7 +477,9 @@ STAGE PLANS:
table name: default.cmv_mat_view2_n0
Stage: Stage-4
- Materialized View Work
+ Materialized View Update
+ name: default.cmv_mat_view2_n0
+ updateCreationMetadata: true
Stage: Stage-1
Pre-Insert task
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out
index 174e132..22b0a90 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out
@@ -275,9 +275,9 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
- Alter Materialized View
+ Alter Materialized View Rewrite
name: default.cmv_mat_view2_n4
- operation: UPDATE_REWRITE_FLAG
+ enable: true
PREHOOK: query: alter materialized view cmv_mat_view2_n4 enable rewrite
PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out
index d050706..b68d98b 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out
@@ -227,7 +227,9 @@ STAGE PLANS:
Table: default.cmv_mat_view
Stage: Stage-5
- Materialized View Work
+ Materialized View Update
+ name: default.cmv_mat_view
+ retrieveAndInclude: true
Stage: Stage-0
Move Operator
@@ -673,7 +675,9 @@ STAGE PLANS:
Table: default.cmv_mat_view
Stage: Stage-4
- Materialized View Work
+ Materialized View Update
+ name: default.cmv_mat_view
+ updateCreationMetadata: true
PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view REBUILD
PREHOOK: type: QUERY
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out
index 13d5919..2ebcb34 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_4.q.out
@@ -222,7 +222,8 @@ STAGE PLANS:
Table: default.cmv_mat_view_n5
Stage: Stage-5
- Materialized View Work
+ Materialized View Update
+ name: default.cmv_mat_view_n5
Stage: Stage-0
Move Operator
@@ -469,9 +470,9 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
- Alter Materialized View
+ Alter Materialized View Rewrite
name: default.cmv_mat_view_n5
- operation: UPDATE_REWRITE_FLAG
+ enable: true
PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view_n5 ENABLE REWRITE
PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE
@@ -914,7 +915,9 @@ STAGE PLANS:
Basic Stats Work:
Stage: Stage-6
- Materialized View Work
+ Materialized View Update
+ name: default.cmv_mat_view_n5
+ updateCreationMetadata: true
Stage: Stage-1
Move Operator
@@ -1222,7 +1225,9 @@ STAGE PLANS:
Table: default.cmv_mat_view_n5
Stage: Stage-4
- Materialized View Work
+ Materialized View Update
+ name: default.cmv_mat_view_n5
+ updateCreationMetadata: true
PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view_n5 REBUILD
PREHOOK: type: QUERY
@@ -1507,7 +1512,9 @@ STAGE PLANS:
Table: default.cmv_mat_view_n5
Stage: Stage-4
- Materialized View Work
+ Materialized View Update
+ name: default.cmv_mat_view_n5
+ updateCreationMetadata: true
PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view_n5 REBUILD
PREHOOK: type: QUERY
@@ -1885,7 +1892,9 @@ STAGE PLANS:
Basic Stats Work:
Stage: Stage-6
- Materialized View Work
+ Materialized View Update
+ name: default.cmv_mat_view_n5
+ updateCreationMetadata: true
Stage: Stage-1
Move Operator
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out
index fdf2772..c821c67 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out
@@ -344,7 +344,9 @@ STAGE PLANS:
Table: default.cmv_mat_view_n6
Stage: Stage-4
- Materialized View Work
+ Materialized View Update
+ name: default.cmv_mat_view_n6
+ updateCreationMetadata: true
PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view_n6 REBUILD
PREHOOK: type: QUERY
@@ -614,7 +616,9 @@ STAGE PLANS:
Table: default.cmv_mat_view_n6
Stage: Stage-4
- Materialized View Work
+ Materialized View Update
+ name: default.cmv_mat_view_n6
+ updateCreationMetadata: true
PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view_n6 REBUILD
PREHOOK: type: QUERY
@@ -840,7 +844,9 @@ STAGE PLANS:
Table: default.cmv_mat_view_n6
Stage: Stage-4
- Materialized View Work
+ Materialized View Update
+ name: default.cmv_mat_view_n6
+ updateCreationMetadata: true
PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view_n6 REBUILD
PREHOOK: type: QUERY
@@ -1070,7 +1076,9 @@ STAGE PLANS:
Table: default.cmv_mat_view_n6
Stage: Stage-4
- Materialized View Work
+ Materialized View Update
+ name: default.cmv_mat_view_n6
+ updateCreationMetadata: true
PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view_n6 REBUILD
PREHOOK: type: QUERY
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out
index 81f9843..51a4a12 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out
@@ -135,6 +135,25 @@ POSTHOOK: Input: default@cmv_mat_view2
#### A masked pattern was here ####
3 9.80
3 978.76
+PREHOOK: query: explain
+alter materialized view cmv_mat_view2 disable rewrite
+PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE
+PREHOOK: Input: default@cmv_mat_view2
+PREHOOK: Output: default@cmv_mat_view2
+POSTHOOK: query: explain
+alter materialized view cmv_mat_view2 disable rewrite
+POSTHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE
+POSTHOOK: Input: default@cmv_mat_view2
+POSTHOOK: Output: default@cmv_mat_view2
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Alter Materialized View Rewrite
+ name: default.cmv_mat_view2
+ disable: true
+
PREHOOK: query: alter materialized view cmv_mat_view2 disable rewrite
PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE
PREHOOK: Input: default@cmv_mat_view2
@@ -275,9 +294,9 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
- Alter Materialized View
+ Alter Materialized View Rewrite
name: default.cmv_mat_view2
- operation: UPDATE_REWRITE_FLAG
+ enable: true
PREHOOK: query: alter materialized view cmv_mat_view2 enable rewrite
PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out
index eb4ccd0..e918e2e 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out
@@ -227,7 +227,9 @@ STAGE PLANS:
Table: default.cmv_mat_view_n1
Stage: Stage-5
- Materialized View Work
+ Materialized View Update
+ name: default.cmv_mat_view_n1
+ retrieveAndInclude: true
Stage: Stage-0
Move Operator
@@ -673,7 +675,9 @@ STAGE PLANS:
Table: default.cmv_mat_view_n1
Stage: Stage-4
- Materialized View Work
+ Materialized View Update
+ name: default.cmv_mat_view_n1
+ updateCreationMetadata: true
PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view_n1 REBUILD
PREHOOK: type: QUERY
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out
index 4bcacfe..5f3d007 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_time_window.q.out
@@ -228,7 +228,8 @@ STAGE PLANS:
Table: default.cmv_mat_view_n3
Stage: Stage-5
- Materialized View Work
+ Materialized View Update
+ name: default.cmv_mat_view_n3
Stage: Stage-0
Move Operator
@@ -469,9 +470,9 @@ STAGE DEPENDENCIES:
STAGE PLANS:
Stage: Stage-0
- Alter Materialized View
+ Alter Materialized View Rewrite
name: default.cmv_mat_view_n3
- operation: UPDATE_REWRITE_FLAG
+ enable: true
PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view_n3 ENABLE REWRITE
PREHOOK: type: ALTER_MATERIALIZED_VIEW_REWRITE
@@ -792,7 +793,9 @@ STAGE PLANS:
Table: default.cmv_mat_view_n3
Stage: Stage-4
- Materialized View Work
+ Materialized View Update
+ name: default.cmv_mat_view_n3
+ updateCreationMetadata: true
PREHOOK: query: ALTER MATERIALIZED VIEW cmv_mat_view_n3 REBUILD
PREHOOK: type: QUERY
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out
index d77a5ee..c6a263f 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned.q.out
@@ -137,7 +137,9 @@ STAGE PLANS:
Table: default.partition_mv_1
Stage: Stage-5
- Materialized View Work
+ Materialized View Update
+ name: default.partition_mv_1
+ retrieveAndInclude: true
PREHOOK: query: CREATE MATERIALIZED VIEW partition_mv_1 PARTITIONED ON (key) AS
SELECT value, key FROM src_txn where key > 200 and key < 250
@@ -701,7 +703,9 @@ STAGE PLANS:
Table: default.partition_mv_1
Stage: Stage-4
- Materialized View Work
+ Materialized View Update
+ name: default.partition_mv_1
+ updateCreationMetadata: true
PREHOOK: query: ALTER MATERIALIZED VIEW partition_mv_1 REBUILD
PREHOOK: type: QUERY
@@ -1059,7 +1063,9 @@ STAGE PLANS:
Table: default.partition_mv_3
Stage: Stage-4
- Materialized View Work
+ Materialized View Update
+ name: default.partition_mv_3
+ updateCreationMetadata: true
PREHOOK: query: ALTER MATERIALIZED VIEW partition_mv_3 REBUILD
PREHOOK: type: QUERY
diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_3.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_3.q.out
index aefc67e..6f1f440 100644
--- a/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_3.q.out
+++ b/ql/src/test/results/clientpositive/llap/materialized_view_partitioned_3.q.out
@@ -150,5 +150,7 @@ STAGE PLANS:
Table: default.partition_mv_sdp
Stage: Stage-5
- Materialized View Work
+ Materialized View Update
+ name: default.partition_mv_sdp
+ retrieveAndInclude: true
diff --git a/ql/src/test/results/clientpositive/masking_mv.q.out b/ql/src/test/results/clientpositive/masking_mv.q.out
index 87e873c..99146ac 100644
--- a/ql/src/test/results/clientpositive/masking_mv.q.out
+++ b/ql/src/test/results/clientpositive/masking_mv.q.out
@@ -117,7 +117,9 @@ STAGE PLANS:
Table: default.masking_test_view_n_mv
Stage: Stage-9
- Materialized View Work
+ Materialized View Update
+ name: default.masking_test_view_n_mv
+ retrieveAndInclude: true
Stage: Stage-3
Merge File Operator
@@ -789,7 +791,9 @@ STAGE PLANS:
Table: default.masking_test_view_n_mv_2
Stage: Stage-9
- Materialized View Work
+ Materialized View Update
+ name: default.masking_test_view_n_mv_2
+ retrieveAndInclude: true
Stage: Stage-3
Merge File Operator