You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by bl...@apache.org on 2019/05/23 03:59:53 UTC
[flink] branch master updated: [FLINK-12582][table][hive]
Alteration APIs in catalogs should check existing object and new object are
of the same class
This is an automated email from the ASF dual-hosted git repository.
bli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new aca9c01 [FLINK-12582][table][hive] Alteration APIs in catalogs should check existing object and new object are of the same class
aca9c01 is described below
commit aca9c018a4b26e50333c33426e77323cbb9b7960
Author: bowen.li <bo...@gmail.com>
AuthorDate: Wed May 22 14:34:42 2019 -0700
[FLINK-12582][table][hive] Alteration APIs in catalogs should check existing object and new object are of the same class
This PR supports alterations in catalogs to check existing object and new object are of the same class.
Most of them currently don't, e.g. you can alter an existing generic table with a new hive table in GenericInMemoryCatalog.
This closes #8514.
---
.../flink/table/catalog/hive/HiveCatalog.java | 55 ++++----
.../hive/HiveCatalogGenericMetadataTest.java | 16 ---
.../catalog/hive/HiveCatalogHiveMetadataTest.java | 3 +
.../table/catalog/GenericInMemoryCatalog.java | 41 +++++-
.../table/catalog/GenericInMemoryCatalogTest.java | 40 +++---
.../flink/table/catalog/CatalogTestBase.java | 155 ++++++++++++++++++---
6 files changed, 218 insertions(+), 92 deletions(-)
diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveCatalog.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveCatalog.java
index d6818f7..22fe08b 100644
--- a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveCatalog.java
+++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveCatalog.java
@@ -27,8 +27,6 @@ import org.apache.flink.table.catalog.CatalogDatabase;
import org.apache.flink.table.catalog.CatalogFunction;
import org.apache.flink.table.catalog.CatalogPartition;
import org.apache.flink.table.catalog.CatalogPartitionSpec;
-import org.apache.flink.table.catalog.CatalogTable;
-import org.apache.flink.table.catalog.CatalogView;
import org.apache.flink.table.catalog.GenericCatalogDatabase;
import org.apache.flink.table.catalog.GenericCatalogFunction;
import org.apache.flink.table.catalog.GenericCatalogTable;
@@ -230,14 +228,27 @@ public class HiveCatalog implements Catalog {
checkArgument(!StringUtils.isNullOrWhitespaceOnly(databaseName), "databaseName cannot be null or empty");
checkNotNull(newDatabase, "newDatabase cannot be null");
+ CatalogDatabase existingDatabase;
+ try {
+ existingDatabase = getDatabase(databaseName);
+ } catch (DatabaseNotExistException e) {
+ if (!ignoreIfNotExists) {
+ throw e;
+ }
+ return;
+ }
+
+ if (existingDatabase.getClass() != newDatabase.getClass()) {
+ throw new CatalogException(
+ String.format("Database types don't match. Existing database is '%s' and new database is '%s'.",
+ existingDatabase.getClass().getName(), newDatabase.getClass().getName())
+ );
+ }
+
Database newHiveDatabase = instantiateHiveDatabase(databaseName, newDatabase);
try {
- if (databaseExists(databaseName)) {
- client.alterDatabase(databaseName, newHiveDatabase);
- } else if (!ignoreIfNotExists) {
- throw new DatabaseNotExistException(catalogName, databaseName);
- }
+ client.alterDatabase(databaseName, newHiveDatabase);
} catch (TException e) {
throw new CatalogException(String.format("Failed to alter database %s", databaseName), e);
}
@@ -360,32 +371,22 @@ public class HiveCatalog implements Catalog {
checkNotNull(tablePath, "tablePath cannot be null");
checkNotNull(newCatalogTable, "newCatalogTable cannot be null");
- if (!tableExists(tablePath)) {
+ Table hiveTable;
+ try {
+ hiveTable = getHiveTable(tablePath);
+ } catch (TableNotExistException e) {
if (!ignoreIfNotExists) {
- throw new TableNotExistException(catalogName, tablePath);
+ throw e;
}
return;
}
- Table oldTable = getHiveTable(tablePath);
- TableType oldTableType = TableType.valueOf(oldTable.getTableType());
+ CatalogBaseTable existingTable = instantiateHiveCatalogTable(hiveTable);
- if (oldTableType == TableType.VIRTUAL_VIEW) {
- if (!(newCatalogTable instanceof CatalogView)) {
- throw new CatalogException(
- String.format("Table types don't match. The existing table is a view, but the new catalog base table is not."));
- }
- // Else, do nothing
- } else if ((oldTableType == TableType.MANAGED_TABLE)) {
- if (!(newCatalogTable instanceof CatalogTable)) {
- throw new CatalogException(
- String.format("Table types don't match. The existing table is a table, but the new catalog base table is not."));
- }
- // Else, do nothing
- } else {
+ if (existingTable.getClass() != newCatalogTable.getClass()) {
throw new CatalogException(
- String.format("Hive table type '%s' is not supported yet.",
- oldTableType.name()));
+ String.format("Table types don't match. Existing table is '%s' and new table is '%s'.",
+ existingTable.getClass().getName(), newCatalogTable.getClass().getName()));
}
Table newTable = instantiateHiveTable(tablePath, newCatalogTable);
@@ -393,7 +394,7 @@ public class HiveCatalog implements Catalog {
// client.alter_table() requires a valid location
// thus, if new table doesn't have that, it reuses location of the old table
if (!newTable.getSd().isSetLocation()) {
- newTable.getSd().setLocation(oldTable.getSd().getLocation());
+ newTable.getSd().setLocation(hiveTable.getSd().getLocation());
}
try {
diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogGenericMetadataTest.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogGenericMetadataTest.java
index f690fc1..ae5f5c89 100644
--- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogGenericMetadataTest.java
+++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogGenericMetadataTest.java
@@ -32,7 +32,6 @@ import org.apache.flink.table.catalog.GenericCatalogDatabase;
import org.apache.flink.table.catalog.GenericCatalogFunction;
import org.apache.flink.table.catalog.GenericCatalogTable;
import org.apache.flink.table.catalog.GenericCatalogView;
-import org.apache.flink.table.catalog.exceptions.CatalogException;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -93,21 +92,6 @@ public class HiveCatalogGenericMetadataTest extends CatalogTestBase {
checkEquals(table, (CatalogTable) catalog.getTable(path1));
}
- // ------ functions ------
-
- @Test
- public void testAlterFunction_differentTypedFunction() throws Exception {
- catalog.createDatabase(db1, createDb(), false);
- catalog.createFunction(path1, createFunction(), false);
-
- exception.expect(CatalogException.class);
- exception.expectMessage(
- "Function types don't match. " +
- "Existing function is 'org.apache.flink.table.catalog.GenericCatalogFunction' and " +
- "new function is 'org.apache.flink.table.catalog.CatalogTestBase$TestFunction'.");
- catalog.alterFunction(path1, new TestFunction(), false);
- }
-
// ------ test utils ------
@Override
diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogHiveMetadataTest.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogHiveMetadataTest.java
index 3b221c0..6699811 100644
--- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogHiveMetadataTest.java
+++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogHiveMetadataTest.java
@@ -67,6 +67,9 @@ public class HiveCatalogHiveMetadataTest extends CatalogTestBase {
public void testAlterFunction() throws Exception {
}
+ public void testAlterFunction_differentTypedFunction() throws Exception {
+ }
+
public void testAlterFunction_FunctionNotExistException() throws Exception {
}
diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/catalog/GenericInMemoryCatalog.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/catalog/GenericInMemoryCatalog.java
index 64d5a8c..c90e1722 100644
--- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/catalog/GenericInMemoryCatalog.java
+++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/catalog/GenericInMemoryCatalog.java
@@ -149,7 +149,16 @@ public class GenericInMemoryCatalog implements Catalog {
checkArgument(!StringUtils.isNullOrWhitespaceOnly(databaseName));
checkNotNull(newDatabase);
- if (databaseExists(databaseName)) {
+ CatalogDatabase existingDatabase = databases.get(databaseName);
+
+ if (existingDatabase != null) {
+ if (existingDatabase.getClass() != newDatabase.getClass()) {
+ throw new CatalogException(
+ String.format("Database types don't match. Existing database is '%s' and new database is '%s'.",
+ existingDatabase.getClass().getName(), newDatabase.getClass().getName())
+ );
+ }
+
databases.put(databaseName, newDatabase.copy());
} else if (!ignoreIfNotExists) {
throw new DatabaseNotExistException(catalogName, databaseName);
@@ -212,13 +221,13 @@ public class GenericInMemoryCatalog implements Catalog {
checkNotNull(tablePath);
checkNotNull(newTable);
- if (tableExists(tablePath)) {
- CatalogBaseTable oldTable = tables.get(tablePath);
+ CatalogBaseTable existingTable = tables.get(tablePath);
- if (oldTable.getClass() != newTable.getClass()) {
+ if (existingTable != null) {
+ if (existingTable.getClass() != newTable.getClass()) {
throw new CatalogException(
- String.format("Table classes don't match. Existing table is '%s' and new table is '%s'. They should be of the same class.",
- oldTable.getClass().getName(), newTable.getClass().getName()));
+ String.format("Table types don't match. Existing table is '%s' and new table is '%s'.",
+ existingTable.getClass().getName(), newTable.getClass().getName()));
}
tables.put(tablePath, newTable.copy());
@@ -362,7 +371,16 @@ public class GenericInMemoryCatalog implements Catalog {
checkNotNull(functionPath);
checkNotNull(newFunction);
- if (functionExists(functionPath)) {
+ CatalogFunction existingFunction = functions.get(functionPath);
+
+ if (existingFunction != null) {
+ if (existingFunction.getClass() != newFunction.getClass()) {
+ throw new CatalogException(
+ String.format("Function types don't match. Existing function is '%s' and new function is '%s'.",
+ existingFunction.getClass().getName(), newFunction.getClass().getName())
+ );
+ }
+
functions.put(functionPath, newFunction.copy());
} else if (!ignoreIfNotExists) {
throw new FunctionNotExistException(catalogName, functionPath);
@@ -464,6 +482,15 @@ public class GenericInMemoryCatalog implements Catalog {
checkNotNull(newPartition);
if (partitionExists(tablePath, partitionSpec)) {
+ CatalogPartition existingPartition = partitions.get(tablePath).get(partitionSpec);
+
+ if (existingPartition.getClass() != newPartition.getClass()) {
+ throw new CatalogException(
+ String.format("Partition types don't match. Existing partition is '%s' and new partition is '%s'.",
+ existingPartition.getClass().getName(), newPartition.getClass().getName())
+ );
+ }
+
partitions.get(tablePath).put(partitionSpec, newPartition.copy());
} else if (!ignoreIfNotExists) {
throw new PartitionNotExistException(catalogName, tablePath, partitionSpec);
diff --git a/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/catalog/GenericInMemoryCatalogTest.java b/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/catalog/GenericInMemoryCatalogTest.java
index 342eb14..d332456 100644
--- a/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/catalog/GenericInMemoryCatalogTest.java
+++ b/flink-table/flink-table-api-java/src/test/java/org/apache/flink/table/catalog/GenericInMemoryCatalogTest.java
@@ -97,28 +97,6 @@ public class GenericInMemoryCatalogTest extends CatalogTestBase {
assertFalse(catalog.partitionExists(path1, catalogPartitionSpec));
}
- @Test
- public void testAlterTable_alterTableWithView() throws Exception {
- catalog.createDatabase(db1, createDb(), false);
- catalog.createTable(path1, createTable(), false);
-
- exception.expect(CatalogException.class);
- exception.expectMessage("Existing table is 'org.apache.flink.table.catalog.GenericCatalogTable' " +
- "and new table is 'org.apache.flink.table.catalog.GenericCatalogView'. They should be of the same class.");
- catalog.alterTable(path1, createView(), false);
- }
-
- @Test
- public void testAlterTable_alterViewWithTable() throws Exception {
- catalog.createDatabase(db1, createDb(), false);
- catalog.createTable(path1, createView(), false);
-
- exception.expect(CatalogException.class);
- exception.expectMessage("Existing table is 'org.apache.flink.table.catalog.GenericCatalogView' " +
- "and new table is 'org.apache.flink.table.catalog.GenericCatalogTable'. They should be of the same class.");
- catalog.alterTable(path1, createTable(), false);
- }
-
// ------ partitions ------
@Test
@@ -310,6 +288,24 @@ public class GenericInMemoryCatalogTest extends CatalogTestBase {
}
@Test
+ public void testAlterPartition_differentTypedPartition() throws Exception {
+ catalog.createDatabase(db1, createDb(), false);
+ catalog.createTable(path1, createPartitionedTable(), false);
+
+ CatalogPartitionSpec partitionSpec = createPartitionSpec();
+ CatalogPartition partition = createPartition();
+ catalog.createPartition(path1, partitionSpec, partition, false);
+
+ exception.expect(CatalogException.class);
+ exception.expectMessage(
+ String.format("Partition types don't match. " +
+ "Existing partition is '%s' and " +
+ "new partition is 'org.apache.flink.table.catalog.CatalogTestBase$TestPartition'.",
+ partition.getClass().getName()));
+ catalog.alterPartition(path1, partitionSpec, new TestPartition(), false);
+ }
+
+ @Test
public void testAlterPartition_PartitionNotExistException_TableNotExist() throws Exception {
catalog.createDatabase(db1, createDb(), false);
diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/catalog/CatalogTestBase.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/catalog/CatalogTestBase.java
index 1e97947..5bfb98a 100644
--- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/catalog/CatalogTestBase.java
+++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/catalog/CatalogTestBase.java
@@ -200,6 +200,19 @@ public abstract class CatalogTestBase {
}
@Test
+ public void testAlterDb_differentTypedDb() throws Exception {
+ catalog.createDatabase(db1, createDb(), false);
+
+ exception.expect(CatalogException.class);
+ exception.expectMessage(
+ String.format("Database types don't match. " +
+ "Existing database is '%s' and " +
+ "new database is 'org.apache.flink.table.catalog.CatalogTestBase$TestDatabase'.",
+ createDb().getClass().getName()));
+ catalog.alterDatabase(db1, new TestDatabase(), false);
+ }
+
+ @Test
public void testAlterDb_DatabaseNotExistException() throws Exception {
exception.expect(DatabaseNotExistException.class);
exception.expectMessage("Database nonexistent does not exist in Catalog");
@@ -382,6 +395,22 @@ public abstract class CatalogTestBase {
}
@Test
+ public void testAlterTable_differentTypedTable() throws Exception {
+ catalog.createDatabase(db1, createDb(), false);
+
+ CatalogTable table = createTable();
+ catalog.createTable(path1, table, false);
+
+ exception.expect(CatalogException.class);
+ exception.expectMessage(
+ String.format("Table types don't match. " +
+ "Existing table is '%s' and " +
+ "new table is 'org.apache.flink.table.catalog.CatalogTestBase$TestTable'.",
+ table.getClass().getName()));
+ catalog.alterTable(path1, new TestTable(), false);
+ }
+
+ @Test
public void testAlterTable_TableNotExistException() throws Exception {
exception.expect(TableNotExistException.class);
exception.expectMessage("Table (or view) non.exist does not exist in Catalog");
@@ -397,26 +426,6 @@ public abstract class CatalogTestBase {
}
@Test
- public void testAlterTable_alterTableWithView() throws Exception {
- catalog.createDatabase(db1, createDb(), false);
- catalog.createTable(path1, createTable(), false);
-
- exception.expect(CatalogException.class);
- exception.expectMessage("The existing table is a table, but the new catalog base table is not.");
- catalog.alterTable(path1, createView(), false);
- }
-
- @Test
- public void testAlterTable_alterViewWithTable() throws Exception {
- catalog.createDatabase(db1, createDb(), false);
- catalog.createTable(path1, createView(), false);
-
- exception.expect(CatalogException.class);
- exception.expectMessage("The existing table is a view, but the new catalog base table is not.");
- catalog.alterTable(path1, createTable(), false);
- }
-
- @Test
public void testRenameTable_nonPartitionedTable() throws Exception {
catalog.createDatabase(db1, createDb(), false);
CatalogTable table = createTable();
@@ -656,6 +665,21 @@ public abstract class CatalogTestBase {
}
@Test
+ public void testAlterFunction_differentTypedFunction() throws Exception {
+ catalog.createDatabase(db1, createDb(), false);
+ CatalogFunction function = createFunction();
+ catalog.createFunction(path1, createFunction(), false);
+
+ exception.expect(CatalogException.class);
+ exception.expectMessage(
+ String.format("Function types don't match. " +
+ "Existing function is '%s' and " +
+ "new function is 'org.apache.flink.table.catalog.CatalogTestBase$TestFunction'.",
+ function.getClass().getName()));
+ catalog.alterFunction(path1, new TestFunction(), false);
+ }
+
+ @Test
public void testAlterFunction_FunctionNotExistException() throws Exception {
exception.expect(FunctionNotExistException.class);
exception.expectMessage("Function db1.nonexist does not exist in Catalog");
@@ -865,6 +889,97 @@ public abstract class CatalogTestBase {
}
/**
+ * Test database used to assert on database of different class.
+ */
+ public static class TestDatabase implements CatalogDatabase {
+ @Override
+ public Map<String, String> getProperties() {
+ return null;
+ }
+
+ @Override
+ public String getComment() {
+ return null;
+ }
+
+ @Override
+ public CatalogDatabase copy() {
+ return null;
+ }
+
+ @Override
+ public Optional<String> getDescription() {
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<String> getDetailedDescription() {
+ return Optional.empty();
+ }
+ }
+
+ /**
+ * Test table used to assert on table of different class.
+ */
+ public static class TestTable implements CatalogBaseTable {
+
+ @Override
+ public Map<String, String> getProperties() {
+ return null;
+ }
+
+ @Override
+ public TableSchema getSchema() {
+ return null;
+ }
+
+ @Override
+ public String getComment() {
+ return null;
+ }
+
+ @Override
+ public CatalogBaseTable copy() {
+ return null;
+ }
+
+ @Override
+ public Optional<String> getDescription() {
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<String> getDetailedDescription() {
+ return Optional.empty();
+ }
+ }
+
+ /**
+ * Test partition used to assert on partition of different class.
+ */
+ public static class TestPartition implements CatalogPartition {
+ @Override
+ public Map<String, String> getProperties() {
+ return null;
+ }
+
+ @Override
+ public CatalogPartition copy() {
+ return null;
+ }
+
+ @Override
+ public Optional<String> getDescription() {
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<String> getDetailedDescription() {
+ return Optional.empty();
+ }
+ }
+
+ /**
* Test function used to assert on function of different class.
*/
public static class TestFunction implements CatalogFunction {