You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by bl...@apache.org on 2019/12/17 23:32:36 UTC
[flink] 02/02: [FLINK-15240][hive] is_generic key is missing for
Flink table stored in HiveCatalog
This is an automated email from the ASF dual-hosted git repository.
bli pushed a commit to branch release-1.9
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 4e07b8c371a9130a9316ff9d4a731caac06c8df9
Author: bowen.li <bo...@gmail.com>
AuthorDate: Tue Dec 17 15:31:07 2019 -0800
[FLINK-15240][hive] is_generic key is missing for Flink table stored in HiveCatalog
backport from 1.10
---
.../flink/table/catalog/hive/HiveCatalog.java | 40 ++++++++++++++---
.../flink/connectors/hive/HiveTableSinkTest.java | 22 ++++++++--
.../hive/HiveCatalogGenericMetadataTest.java | 2 +-
.../catalog/hive/HiveCatalogHiveMetadataTest.java | 2 +-
.../table/catalog/hive/HiveCatalogITCase.java | 50 +++++++++++++++++++---
...tBase.java => HiveCatalogMetadataTestBase.java} | 2 +-
6 files changed, 101 insertions(+), 17 deletions(-)
diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveCatalog.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveCatalog.java
index dcfb313..2713923 100644
--- a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveCatalog.java
+++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveCatalog.java
@@ -500,10 +500,21 @@ public class HiveCatalog extends AbstractCatalog {
// Table properties
Map<String, String> properties = hiveTable.getParameters();
- boolean isGeneric = Boolean.valueOf(properties.get(CatalogConfig.IS_GENERIC));
- if (isGeneric) {
- properties = retrieveFlinkProperties(properties);
+ // When retrieving a table, a generic table needs explicitly have a key is_generic = true
+ // otherwise, this is a Hive table if 1) the key is missing 2) is_generic = false
+ // this is opposite to creating a table. See instantiateHiveTable()
+
+ if (!properties.containsKey(CatalogConfig.IS_GENERIC)) {
+ // must be a hive table
+ properties.put(CatalogConfig.IS_GENERIC, String.valueOf(false));
+ } else {
+ boolean isGeneric = Boolean.valueOf(properties.get(CatalogConfig.IS_GENERIC));
+
+ if (isGeneric) {
+ properties = retrieveFlinkProperties(properties);
+ }
}
+
String comment = properties.remove(HiveCatalogConfig.COMMENT);
// Table schema
@@ -555,11 +566,24 @@ public class HiveCatalog extends AbstractCatalog {
properties.put(HiveCatalogConfig.COMMENT, table.getComment());
}
- boolean isGeneric = Boolean.valueOf(properties.get(CatalogConfig.IS_GENERIC));
+ // Make sure there's no null in properties
+ properties = cleanNullProperties(properties);
- if (isGeneric) {
+ // When creating a table, A hive table needs explicitly have a key is_generic = false
+ // otherwise, this is a generic table if 1) the key is missing 2) is_generic = true
+ // this is opposite to reading a table and instantiating a CatalogTable. See instantiateCatalogTable()
+ if (!properties.containsKey(CatalogConfig.IS_GENERIC)) {
+ // must be a generic catalog
+ properties.put(CatalogConfig.IS_GENERIC, String.valueOf(true));
properties = maskFlinkProperties(properties);
+ } else {
+ boolean isGeneric = Boolean.valueOf(properties.get(CatalogConfig.IS_GENERIC));
+
+ if (isGeneric) {
+ properties = maskFlinkProperties(properties);
+ }
}
+
// Table properties
hiveTable.setParameters(properties);
@@ -602,6 +626,12 @@ public class HiveCatalog extends AbstractCatalog {
return hiveTable;
}
+ private static Map<String, String> cleanNullProperties(Map<String, String> properties) {
+ return properties.entrySet().stream()
+ .filter(e -> e.getKey() != null && e.getValue() != null)
+ .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue()));
+ }
+
private static void setStorageFormat(StorageDescriptor sd, Map<String, String> properties) {
// TODO: allow user to specify storage format. Simply use text format for now
String storageFormatName = DEFAULT_HIVE_TABLE_STORAGE_FORMAT;
diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSinkTest.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSinkTest.java
index 48cb741..0bf9504 100644
--- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSinkTest.java
+++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSinkTest.java
@@ -31,6 +31,7 @@ import org.apache.flink.table.catalog.CatalogPartitionSpec;
import org.apache.flink.table.catalog.CatalogTable;
import org.apache.flink.table.catalog.CatalogTableImpl;
import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.catalog.config.CatalogConfig;
import org.apache.flink.table.catalog.hive.HiveCatalog;
import org.apache.flink.table.catalog.hive.HiveTestUtils;
import org.apache.flink.table.sources.InputFormatTableSource;
@@ -281,7 +282,7 @@ public class HiveTableSinkTest {
}
private RowTypeInfo createDestTable(String dbName, String tblName, TableSchema tableSchema, int numPartCols) throws Exception {
- CatalogTable catalogTable = createCatalogTable(tableSchema, numPartCols);
+ CatalogTable catalogTable = createHiveCatalogTable(tableSchema, numPartCols);
hiveCatalog.createTable(new ObjectPath(dbName, tblName), catalogTable, false);
return new RowTypeInfo(tableSchema.getFieldTypes(), tableSchema.getFieldNames());
}
@@ -297,13 +298,26 @@ public class HiveTableSinkTest {
return createDestTable(dbName, tblName, builder.build(), numPartCols);
}
- private CatalogTable createCatalogTable(TableSchema tableSchema, int numPartCols) {
+ private CatalogTable createHiveCatalogTable(TableSchema tableSchema, int numPartCols) {
if (numPartCols == 0) {
- return new CatalogTableImpl(tableSchema, new HashMap<>(), "");
+ return new CatalogTableImpl(
+ tableSchema,
+ new HashMap<String, String>() {{
+ // creating a hive table needs explicit is_generic=false flag
+ put(CatalogConfig.IS_GENERIC, String.valueOf(false));
+ }},
+ "");
}
String[] partCols = new String[numPartCols];
System.arraycopy(tableSchema.getFieldNames(), tableSchema.getFieldNames().length - numPartCols, partCols, 0, numPartCols);
- return new CatalogTableImpl(tableSchema, Arrays.asList(partCols), new HashMap<>(), "");
+ return new CatalogTableImpl(
+ tableSchema,
+ Arrays.asList(partCols),
+ new HashMap<String, String>() {{
+ // creating a hive table needs explicit is_generic=false flag
+ put(CatalogConfig.IS_GENERIC, String.valueOf(false));
+ }},
+ "");
}
private void verifyWrittenData(List<Row> expected, List<String> results) throws Exception {
diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogGenericMetadataTest.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogGenericMetadataTest.java
index b9a52b6..4ede982 100644
--- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogGenericMetadataTest.java
+++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogGenericMetadataTest.java
@@ -26,7 +26,7 @@ import org.junit.Test;
/**
* Test for HiveCatalog on generic metadata.
*/
-public class HiveCatalogGenericMetadataTest extends HiveCatalogTestBase {
+public class HiveCatalogGenericMetadataTest extends HiveCatalogMetadataTestBase {
@BeforeClass
public static void init() {
diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogHiveMetadataTest.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogHiveMetadataTest.java
index 0fe10d7..ed3c7a2 100644
--- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogHiveMetadataTest.java
+++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogHiveMetadataTest.java
@@ -46,7 +46,7 @@ import static org.junit.Assert.assertFalse;
/**
* Test for HiveCatalog on Hive metadata.
*/
-public class HiveCatalogHiveMetadataTest extends HiveCatalogTestBase {
+public class HiveCatalogHiveMetadataTest extends HiveCatalogMetadataTestBase {
@BeforeClass
public static void init() {
diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogITCase.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogITCase.java
index 0898ac3..9e28001 100644
--- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogITCase.java
+++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogITCase.java
@@ -28,6 +28,7 @@ import org.apache.flink.table.api.java.BatchTableEnvironment;
import org.apache.flink.table.catalog.CatalogTable;
import org.apache.flink.table.catalog.CatalogTableBuilder;
import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.catalog.exceptions.TableNotExistException;
import org.apache.flink.table.descriptors.FileSystem;
import org.apache.flink.table.descriptors.FormatDescriptor;
import org.apache.flink.table.descriptors.OldCsv;
@@ -70,8 +71,12 @@ public class HiveCatalogITCase {
private static HiveCatalog hiveCatalog;
private static HiveConf hiveConf;
- private String sourceTableName = "csv_source";
- private String sinkTableName = "csv_sink";
+ private static String sourceTableName = "csv_source";
+ private static String sinkTableName = "csv_sink";
+ private static ObjectPath source = new ObjectPath("default", sourceTableName);
+ private static ObjectPath sink = new ObjectPath("default", sinkTableName);
+
+ private String testFilePath = this.getClass().getResource("/csv/test.csv").getPath();
@BeforeClass
public static void createCatalog() {
@@ -81,14 +86,49 @@ public class HiveCatalogITCase {
}
@AfterClass
- public static void closeCatalog() {
+ public static void closeCatalog() throws TableNotExistException {
+ hiveCatalog.dropTable(source, true);
+ hiveCatalog.dropTable(sink, true);
+
if (hiveCatalog != null) {
hiveCatalog.close();
}
}
@Test
- public void testGenericTable() throws Exception {
+ public void testCsvTableViaDDL() throws Exception {
+ ExecutionEnvironment execEnv = ExecutionEnvironment.createLocalEnvironment(1);
+ BatchTableEnvironment tableEnv = BatchTableEnvironment.create(execEnv);
+
+ tableEnv.registerCatalog("myhive", hiveCatalog);
+ tableEnv.useCatalog("myhive");
+
+ tableEnv.sqlUpdate("create table myhive.`default`.test2 (name String, age Int) with (\n" +
+ " 'connector.type' = 'filesystem',\n" +
+ " 'connector.path' = 'file://" + testFilePath + "',\n" +
+ " 'format.type' = 'csv',\n" +
+ " 'format.fields.0.name' = 'name',\n" +
+ " 'format.fields.0.type' = 'STRING',\n" +
+ " 'format.fields.1.name' = 'age',\n" +
+ " 'format.fields.1.type' = 'INT'\n" +
+ ")");
+
+ Table t = tableEnv.sqlQuery("select * from test2");
+
+ List<Row> result = tableEnv.toDataSet(t, Row.class).collect();
+
+ // assert query result
+ assertEquals(
+ Arrays.asList(
+ Row.of("1", 1),
+ Row.of("2", 2),
+ Row.of("3", 3)),
+ result
+ );
+ }
+
+ @Test
+ public void testCsvTableViaAPI() throws Exception {
ExecutionEnvironment execEnv = ExecutionEnvironment.createLocalEnvironment(1);
BatchTableEnvironment tableEnv = BatchTableEnvironment.create(execEnv);
@@ -105,7 +145,7 @@ public class HiveCatalogITCase {
CatalogTable source =
new CatalogTableBuilder(
- new FileSystem().path(this.getClass().getResource("/csv/test.csv").getPath()),
+ new FileSystem().path(testFilePath),
schema)
.withFormat(format)
.inAppendMode()
diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogTestBase.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogMetadataTestBase.java
similarity index 95%
rename from flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogTestBase.java
rename to flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogMetadataTestBase.java
index 232c250..040131d 100644
--- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogTestBase.java
+++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogMetadataTestBase.java
@@ -26,7 +26,7 @@ import org.junit.Test;
/**
* Base class for testing HiveCatalog.
*/
-public abstract class HiveCatalogTestBase extends CatalogTestBase {
+public abstract class HiveCatalogMetadataTestBase extends CatalogTestBase {
// ------ functions ------