You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kr...@apache.org on 2022/01/28 05:24:43 UTC
[hive] branch master updated: HIVE-25899: Materialized view registry does not clean dropped views (Krisztian Kasa, reviewed by Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new ced8822 HIVE-25899: Materialized view registry does not clean dropped views (Krisztian Kasa, reviewed by Zoltan Haindrich)
ced8822 is described below
commit ced8822c231b6c164e72dbf6c236d1d52e8b5591
Author: Krisztian Kasa <ka...@gmail.com>
AuthorDate: Fri Jan 28 06:23:11 2022 +0100
HIVE-25899: Materialized view registry does not clean dropped views (Krisztian Kasa, reviewed by Zoltan Haindrich)
---
.../ql/metadata/HiveMaterializedViewsRegistry.java | 5 +++
.../hive/ql/metadata/MaterializedViewsCache.java | 39 ++++++++++++++++++----
.../hadoop/hive/ql/parse/CalcitePlanner.java | 2 ++
.../ql/metadata/TestMaterializedViewsCache.java | 10 ++++++
4 files changed, 49 insertions(+), 7 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
index 8331ad6..a4c318a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
@@ -372,6 +372,11 @@ public final class HiveMaterializedViewsRegistry {
return materializedViewsCache.get(querySql);
}
+ public boolean isEmpty() {
+ return materializedViewsCache.isEmpty();
+ }
+
+
private static RelNode createMaterializedViewScan(HiveConf conf, Table viewTable) {
// 0. Recreate cluster
final RelOptPlanner planner = CalcitePlanner.createPlanner(conf);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/MaterializedViewsCache.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/MaterializedViewsCache.java
index a5e109e..f1b0845 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/MaterializedViewsCache.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/MaterializedViewsCache.java
@@ -124,30 +124,51 @@ public class MaterializedViewsCache {
// Delete only if the create time for the input materialized view table and the table
// in the map match. Otherwise, keep the one in the map.
dbMap.computeIfPresent(materializedViewTable.getTableName(), (mvTableName, oldMaterialization) -> {
- if (HiveMaterializedViewUtils.extractTable(oldMaterialization).equals(materializedViewTable)) {
- List<HiveRelOptMaterialization> materializationList =
- sqlToMaterializedView.get(materializedViewTable.getViewExpandedText());
- materializationList.remove(oldMaterialization);
+ Table oldTable = HiveMaterializedViewUtils.extractTable(oldMaterialization);
+ if (materializedViewTable.equals(oldTable)) {
+ remove(oldMaterialization, oldTable);
return null;
}
return oldMaterialization;
});
+
+ if (dbMap.isEmpty()) {
+ materializedViews.remove(materializedViewTable.getDbName());
+ }
}
LOG.debug("Materialized view {}.{} removed from registry",
materializedViewTable.getDbName(), materializedViewTable.getTableName());
}
+ private void remove(HiveRelOptMaterialization materialization, Table mvTable) {
+ if (mvTable == null) {
+ return;
+ }
+
+ List<HiveRelOptMaterialization> materializationList = sqlToMaterializedView.get(mvTable.getViewExpandedText());
+ if (materializationList == null) {
+ return;
+ }
+
+ materializationList.remove(materialization);
+ if (materializationList.isEmpty()) {
+ sqlToMaterializedView.remove(mvTable.getViewExpandedText());
+ }
+ }
+
public void remove(String dbName, String tableName) {
ConcurrentMap<String, HiveRelOptMaterialization> dbMap = materializedViews.get(dbName);
if (dbMap != null) {
dbMap.computeIfPresent(tableName, (mvTableName, materialization) -> {
- String queryText = HiveMaterializedViewUtils.extractTable(materialization).getViewExpandedText();
- List<HiveRelOptMaterialization> materializationList = sqlToMaterializedView.get(queryText);
- materializationList.remove(materialization);
+ remove(materialization, HiveMaterializedViewUtils.extractTable(materialization));
return null;
});
+ if (dbMap.isEmpty()) {
+ materializedViews.remove(dbName);
+ }
+
LOG.debug("Materialized view {}.{} removed from registry", dbName, tableName);
}
}
@@ -180,4 +201,8 @@ public class MaterializedViewsCache {
relOptMaterializationList.size());
return unmodifiableList(relOptMaterializationList);
}
+
+ public boolean isEmpty() {
+ return materializedViews.isEmpty();
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index ab4506e..80687d8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -151,6 +151,7 @@ import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.HiveMaterializedViewsRegistry;
import org.apache.hadoop.hive.ql.metadata.HiveRelOptMaterialization;
import org.apache.hadoop.hive.ql.metadata.HiveUtils;
import org.apache.hadoop.hive.ql.metadata.NotNullConstraint;
@@ -2074,6 +2075,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
private boolean isMaterializedViewRewritingByTextEnabled() {
return conf.getBoolVar(ConfVars.HIVE_MATERIALIZED_VIEW_ENABLE_AUTO_REWRITING_SQL) &&
+ !HiveMaterializedViewsRegistry.get().isEmpty() &&
mvRebuildMode == MaterializationRebuildMode.NONE &&
!rootQB.isMaterializedView() && !ctx.isLoadingMaterializedView() && !rootQB.isCTAS() &&
rootQB.getIsQuery() &&
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestMaterializedViewsCache.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestMaterializedViewsCache.java
index 122976d..da532e6 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestMaterializedViewsCache.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestMaterializedViewsCache.java
@@ -92,6 +92,7 @@ class TestMaterializedViewsCache {
MaterializedViewsCache emptyCache = new MaterializedViewsCache();
assertThat(emptyCache.get("select 'any definition'").isEmpty(), is(true));
+ assertThat(emptyCache.isEmpty(), is(true));
assertThat(emptyCache.values().isEmpty(), is(true));
}
@@ -117,6 +118,7 @@ class TestMaterializedViewsCache {
assertThat(materializedViewsCache.get(defaultMV1.getDbName(), defaultMV1.getTableName()), is(defaultMaterialization1));
assertThat(materializedViewsCache.get(defaultMV1.getViewExpandedText()).size(), is(1));
assertThat(materializedViewsCache.get(defaultMV1.getViewExpandedText()).get(0), is(defaultMaterialization1));
+ assertThat(materializedViewsCache.isEmpty(), is(false));
assertThat(materializedViewsCache.values().size(), is(1));
assertThat(materializedViewsCache.values().get(0), is(defaultMaterialization1));
}
@@ -158,6 +160,7 @@ class TestMaterializedViewsCache {
assertThat(materializedViewsCache.get(defaultMV1.getViewExpandedText()).size(), is(2));
assertThat(materializedViewsCache.get(defaultMV1.getViewExpandedText()), hasItem(defaultMaterialization1));
assertThat(materializedViewsCache.get(defaultMV1.getViewExpandedText()), hasItem(defaultMaterialization1Same));
+ assertThat(materializedViewsCache.isEmpty(), is(false));
assertThat(materializedViewsCache.values().size(), is(2));
}
@@ -171,6 +174,7 @@ class TestMaterializedViewsCache {
assertThat(materializedViewsCache.get(defaultMV1.getViewExpandedText()).size(), is(2));
assertThat(materializedViewsCache.get(defaultMV1.getViewExpandedText()), hasItem(defaultMaterialization1));
assertThat(materializedViewsCache.get(defaultMV1.getViewExpandedText()), hasItem(db1Materialization1));
+ assertThat(materializedViewsCache.isEmpty(), is(false));
assertThat(materializedViewsCache.values().size(), is(2));
}
@@ -181,6 +185,7 @@ class TestMaterializedViewsCache {
assertThat(materializedViewsCache.get(defaultMV1.getViewExpandedText()).size(), is(1));
assertThat(materializedViewsCache.get(defaultMV1.getViewExpandedText()), hasItem(defaultMaterialization1));
+ assertThat(materializedViewsCache.isEmpty(), is(false));
assertThat(materializedViewsCache.values().size(), is(2));
}
@@ -191,6 +196,7 @@ class TestMaterializedViewsCache {
assertThat(materializedViewsCache.get(defaultMV1.getDbName(), defaultMV1.getTableName()), is(defaultMaterialization1));
assertThat(materializedViewsCache.get(defaultMV1.getViewExpandedText()).size(), is(1));
assertThat(materializedViewsCache.get(defaultMV1.getViewExpandedText()), hasItem(defaultMaterialization1));
+ assertThat(materializedViewsCache.isEmpty(), is(false));
assertThat(materializedViewsCache.values().size(), is(1));
assertThat(materializedViewsCache.values(), hasItem(defaultMaterialization1));
}
@@ -205,6 +211,7 @@ class TestMaterializedViewsCache {
assertThat(materializedViewsCache.get(defaultMV1.getDbName(), defaultMV1.getTableName()), is(newMaterialization));
assertThat(materializedViewsCache.get(defaultMV1.getViewExpandedText()).size(), is(1));
assertThat(materializedViewsCache.get(defaultMV1.getViewExpandedText()), hasItem(newMaterialization));
+ assertThat(materializedViewsCache.isEmpty(), is(false));
assertThat(materializedViewsCache.values().size(), is(1));
assertThat(materializedViewsCache.values(), hasItem(newMaterialization));
}
@@ -217,6 +224,7 @@ class TestMaterializedViewsCache {
assertThat(materializedViewsCache.get(defaultMV1.getDbName(), defaultMV1.getTableName()), is(defaultMaterialization1));
assertThat(materializedViewsCache.get(defaultMV1.getViewExpandedText()).size(), is(1));
assertThat(materializedViewsCache.get(defaultMV1.getViewExpandedText()), hasItem(defaultMaterialization1));
+ assertThat(materializedViewsCache.isEmpty(), is(false));
assertThat(materializedViewsCache.values().size(), is(1));
assertThat(materializedViewsCache.values(), hasItem(defaultMaterialization1));
}
@@ -229,6 +237,7 @@ class TestMaterializedViewsCache {
assertThat(materializedViewsCache.get(defaultMV1.getDbName(), defaultMV1.getTableName()), is(nullValue()));
assertThat(materializedViewsCache.get(defaultMV1.getViewExpandedText()).isEmpty(), is(true));
+ assertThat(materializedViewsCache.isEmpty(), is(true));
assertThat(materializedViewsCache.values().isEmpty(), is(true));
}
@@ -240,6 +249,7 @@ class TestMaterializedViewsCache {
assertThat(materializedViewsCache.get(defaultMV1.getDbName(), defaultMV1.getTableName()), is(nullValue()));
assertThat(materializedViewsCache.get(defaultMV1.getViewExpandedText()).isEmpty(), is(true));
+ assertThat(materializedViewsCache.isEmpty(), is(true));
assertThat(materializedViewsCache.values().isEmpty(), is(true));
}