You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sa...@apache.org on 2023/06/07 10:28:19 UTC
[hive] branch branch-3 updated: HIVE-27379: Backport HIVE-22566 : Drop table involved in materialized view leaves the table in inconsistent state (Pablo Junge, reviewed by Miklos Gergely)
This is an automated email from the ASF dual-hosted git repository.
sankarh pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new e61cd0f12f1 HIVE-27379: Backport HIVE-22566 : Drop table involved in materialized view leaves the table in inconsistent state (Pablo Junge, reviewed by Miklos Gergely)
e61cd0f12f1 is described below
commit e61cd0f12f1be76ebf2b3383d0fbe66af8ab3296
Author: Diksha628 <43...@users.noreply.github.com>
AuthorDate: Wed Jun 7 15:58:12 2023 +0530
HIVE-27379: Backport HIVE-22566 : Drop table involved in materialized view leaves the table in inconsistent state (Pablo Junge, reviewed by Miklos Gergely)
Signed-off-by: Sankar Hariappan <sa...@apache.org>
Closes (#4359)
---
.../hcatalog/listener/DummyRawStoreFailEvent.java | 5 ++
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 7 ---
.../clientnegative/drop_table_used_by_mv2.q | 12 ++++
.../clientnegative/drop_table_used_by_mv.q.out | 3 +-
.../clientnegative/drop_table_used_by_mv2.q.out | 72 ++++++++++++++++++++++
.../hadoop/hive/metastore/HiveMetaStore.java | 9 +++
.../apache/hadoop/hive/metastore/ObjectStore.java | 35 +++++++++++
.../org/apache/hadoop/hive/metastore/RawStore.java | 10 +++
.../hadoop/hive/metastore/cache/CachedStore.java | 5 ++
.../metastore/DummyRawStoreControlledCommit.java | 5 ++
.../metastore/DummyRawStoreForJdoConnection.java | 5 ++
11 files changed, 160 insertions(+), 8 deletions(-)
diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
index 33f2ea09e57..480cdc3125b 100644
--- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
+++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
@@ -261,6 +261,11 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable {
}
}
+ @Override
+ public List<String> isPartOfMaterializedView(String catName, String dbName, String tblName) {
+ return objectStore.isPartOfMaterializedView(catName, dbName, tblName);
+ }
+
@Override
public Table getTable(String catName, String dbName, String tableName) throws MetaException {
return objectStore.getTable(catName, dbName, tableName);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index fc969fc651c..faeeb864a69 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -1039,13 +1039,6 @@ public class Hive {
if (!ignoreUnknownTab) {
throw new HiveException(e);
}
- } catch (MetaException e) {
- int idx = ExceptionUtils.indexOfType(e, SQLIntegrityConstraintViolationException.class);
- if (idx != -1 && ExceptionUtils.getThrowables(e)[idx].getMessage().contains("MV_TABLES_USED")) {
- throw new HiveException("Cannot drop table since it is used by at least one materialized view definition. " +
- "Please drop any materialized view that uses the table before dropping it", e);
- }
- throw new HiveException(e);
} catch (Exception e) {
throw new HiveException(e);
}
diff --git a/ql/src/test/queries/clientnegative/drop_table_used_by_mv2.q b/ql/src/test/queries/clientnegative/drop_table_used_by_mv2.q
new file mode 100644
index 00000000000..458cc9ea942
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/drop_table_used_by_mv2.q
@@ -0,0 +1,12 @@
+create table mytable (key int, value string);
+insert into mytable values (1, 'val1'), (2, 'val2');
+create view myview as select * from mytable;
+
+create materialized view mv1 disable rewrite as
+select key, value from myview;
+create materialized view mv2 disable rewrite as
+select count(*) from myview;
+
+-- dropping the view is fine, as the MV uses not the view itself, but it's query for creating it's own during it's creation
+drop view myview;
+drop table mytable;
diff --git a/ql/src/test/results/clientnegative/drop_table_used_by_mv.q.out b/ql/src/test/results/clientnegative/drop_table_used_by_mv.q.out
index 88e3b7dcdef..d35e9fa976d 100644
--- a/ql/src/test/results/clientnegative/drop_table_used_by_mv.q.out
+++ b/ql/src/test/results/clientnegative/drop_table_used_by_mv.q.out
@@ -32,4 +32,5 @@ PREHOOK: query: drop table mytable
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@mytable
PREHOOK: Output: default@mytable
-FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Cannot drop table since it is used by at least one materialized view definition. Please drop any materialized view that uses the table before dropping it
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Cannot drop table as it is used in the following materialized views [default.mv1]
+)
diff --git a/ql/src/test/results/clientnegative/drop_table_used_by_mv2.q.out b/ql/src/test/results/clientnegative/drop_table_used_by_mv2.q.out
new file mode 100644
index 00000000000..fd6faa5d8dd
--- /dev/null
+++ b/ql/src/test/results/clientnegative/drop_table_used_by_mv2.q.out
@@ -0,0 +1,72 @@
+PREHOOK: query: create table mytable (key int, value string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@mytable
+POSTHOOK: query: create table mytable (key int, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@mytable
+PREHOOK: query: insert into mytable values (1, 'val1'), (2, 'val2')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@mytable
+POSTHOOK: query: insert into mytable values (1, 'val1'), (2, 'val2')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@mytable
+POSTHOOK: Lineage: mytable.key SCRIPT []
+POSTHOOK: Lineage: mytable.value SCRIPT []
+PREHOOK: query: create view myview as select * from mytable
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@mytable
+PREHOOK: Output: database:default
+PREHOOK: Output: default@myview
+POSTHOOK: query: create view myview as select * from mytable
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@mytable
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@myview
+POSTHOOK: Lineage: myview.key SIMPLE [(mytable)mytable.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: myview.value SIMPLE [(mytable)mytable.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: create materialized view mv1 disable rewrite as
+select key, value from myview
+PREHOOK: type: CREATE_MATERIALIZED_VIEW
+PREHOOK: Input: default@mytable
+PREHOOK: Input: default@myview
+PREHOOK: Output: database:default
+PREHOOK: Output: default@mv1
+POSTHOOK: query: create materialized view mv1 disable rewrite as
+select key, value from myview
+POSTHOOK: type: CREATE_MATERIALIZED_VIEW
+POSTHOOK: Input: default@mytable
+POSTHOOK: Input: default@myview
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@mv1
+PREHOOK: query: create materialized view mv2 disable rewrite as
+select count(*) from myview
+PREHOOK: type: CREATE_MATERIALIZED_VIEW
+PREHOOK: Input: default@mytable
+PREHOOK: Input: default@myview
+PREHOOK: Output: database:default
+PREHOOK: Output: default@mv2
+POSTHOOK: query: create materialized view mv2 disable rewrite as
+select count(*) from myview
+POSTHOOK: type: CREATE_MATERIALIZED_VIEW
+POSTHOOK: Input: default@mytable
+POSTHOOK: Input: default@myview
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@mv2
+PREHOOK: query: drop view myview
+PREHOOK: type: DROPVIEW
+PREHOOK: Input: default@myview
+PREHOOK: Output: default@myview
+POSTHOOK: query: drop view myview
+POSTHOOK: type: DROPVIEW
+POSTHOOK: Input: default@myview
+POSTHOOK: Output: default@myview
+PREHOOK: query: drop table mytable
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@mytable
+PREHOOK: Output: default@mytable
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Cannot drop table as it is used in the following materialized views [default.mv1, default.mv2]
+)
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 8270d8bf282..629666ac000 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -2536,6 +2536,15 @@ public class HiveMetaStore extends ThriftHiveMetastore {
if (tbl == null) {
throw new NoSuchObjectException(name + " doesn't exist");
}
+
+ // Check if table is part of a materialized view.
+ // If it is, it cannot be dropped.
+ List<String> isPartOfMV = ms.isPartOfMaterializedView(catName, dbname, name);
+ if (!isPartOfMV.isEmpty()) {
+ throw new MetaException(String.format("Cannot drop table as it is used in the following materialized" +
+ " views %s%n", isPartOfMV));
+ }
+
if (tbl.getSd() == null) {
throw new MetaException("Table metadata is corrupted");
}
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 4f02e7b8325..9b11e4836c3 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -1238,6 +1238,41 @@ public class ObjectStore implements RawStore, Configurable {
return success;
}
+ @Override
+ public List<String> isPartOfMaterializedView(String catName, String dbName, String tblName) {
+
+ boolean committed = false;
+ Query query = null;
+ List<String> mViewList = new ArrayList<>();
+
+ try {
+ openTransaction();
+
+ query = pm.newQuery("select from org.apache.hadoop.hive.metastore.model.MCreationMetadata");
+
+ List<MCreationMetadata> creationMetadata = (List<MCreationMetadata>)query.execute();
+ Iterator<MCreationMetadata> iter = creationMetadata.iterator();
+
+ while (iter.hasNext())
+ {
+ MCreationMetadata p = iter.next();
+ Set<MTable> tables = p.getTables();
+ for (MTable table : tables) {
+ if (dbName.equals(table.getDatabase().getName()) && tblName.equals(table.getTableName())) {
+ LOG.info("Cannot drop table " + table.getTableName() +
+ " as it is being used by MView " + p.getTblName());
+ mViewList.add(p.getDbName() + "." + p.getTblName());
+ }
+ }
+ }
+
+ committed = commitTransaction();
+ } finally {
+ rollbackAndCleanup(committed, query);
+ }
+ return mViewList;
+ }
+
private List<MConstraint> listAllTableConstraintsWithOptionalConstraintName(
String catName, String dbName, String tableName, String constraintname) {
catName = normalizeIdentifier(catName);
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
index 2bd4eb5b688..dba07cd56f9 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
@@ -1672,4 +1672,14 @@ public interface RawStore extends Configurable {
* @param tableName the name of the table for which the dump is being taken
*/
List<WriteEventInfo> getAllWriteEventInfo(long txnId, String dbName, String tableName) throws MetaException;
+
+ /**
+ * Checking if table is part of a materialized view.
+ * @param catName catalog the table is in
+ * @param dbName database the table is in
+ * @param tblName table name
+ * @return list of materialized views that uses the table
+ */
+ List<String> isPartOfMaterializedView(String catName, String dbName, String tblName);
+
}
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
index 39cba08eebf..cde755537a2 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
@@ -2491,6 +2491,11 @@ public class CachedStore implements RawStore, Configurable {
&& MetastoreConf.getAsString(conf, MetastoreConf.ConfVars.CACHED_RAW_STORE_CACHED_OBJECTS_BLACKLIST).isEmpty();
}
+ @Override
+ public List<String> isPartOfMaterializedView(String catName, String dbName, String tblName) {
+ return rawStore.isPartOfMaterializedView(catName, dbName, tblName);
+ }
+
@VisibleForTesting
void resetCatalogCache() {
sharedCache.resetCatalogCache();
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
index e57fcf2d9e3..974a9d61236 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
@@ -1193,6 +1193,11 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable {
objectStore.cleanWriteNotificationEvents(olderThan);
}
+ @Override
+ public List<String> isPartOfMaterializedView(String catName, String dbName, String tblName) {
+ return objectStore.isPartOfMaterializedView(catName, dbName, tblName);
+ }
+
@Override
public List<WriteEventInfo> getAllWriteEventInfo(long txnId, String dbName, String tableName) throws MetaException {
return objectStore.getAllWriteEventInfo(txnId, dbName, tableName);
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
index 46548e5a920..9af8142bda7 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
@@ -1198,6 +1198,11 @@ public class DummyRawStoreForJdoConnection implements RawStore {
public void cleanWriteNotificationEvents(int olderThan) {
}
+ @Override
+ public List<String> isPartOfMaterializedView(String catName, String dbName, String tblName) {
+ throw new RuntimeException("unimplemented");
+ }
+
@Override
public List<WriteEventInfo> getAllWriteEventInfo(long txnId, String dbName, String tableName) throws MetaException {
return null;