You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by bl...@apache.org on 2019/12/17 23:32:36 UTC

[flink] 02/02: [FLINK-15240][hive] is_generic key is missing for Flink table stored in HiveCatalog

This is an automated email from the ASF dual-hosted git repository.

bli pushed a commit to branch release-1.9
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 4e07b8c371a9130a9316ff9d4a731caac06c8df9
Author: bowen.li <bo...@gmail.com>
AuthorDate: Tue Dec 17 15:31:07 2019 -0800

    [FLINK-15240][hive] is_generic key is missing for Flink table stored in HiveCatalog
    
    backport from 1.10
---
 .../flink/table/catalog/hive/HiveCatalog.java      | 40 ++++++++++++++---
 .../flink/connectors/hive/HiveTableSinkTest.java   | 22 ++++++++--
 .../hive/HiveCatalogGenericMetadataTest.java       |  2 +-
 .../catalog/hive/HiveCatalogHiveMetadataTest.java  |  2 +-
 .../table/catalog/hive/HiveCatalogITCase.java      | 50 +++++++++++++++++++---
 ...tBase.java => HiveCatalogMetadataTestBase.java} |  2 +-
 6 files changed, 101 insertions(+), 17 deletions(-)

diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveCatalog.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveCatalog.java
index dcfb313..2713923 100644
--- a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveCatalog.java
+++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/catalog/hive/HiveCatalog.java
@@ -500,10 +500,21 @@ public class HiveCatalog extends AbstractCatalog {
 		// Table properties
 		Map<String, String> properties = hiveTable.getParameters();
 
-		boolean isGeneric = Boolean.valueOf(properties.get(CatalogConfig.IS_GENERIC));
-		if (isGeneric) {
-			properties = retrieveFlinkProperties(properties);
+		// When retrieving a table, a generic table needs explicitly have a key is_generic = true
+		// otherwise, this is a Hive table if 1) the key is missing 2) is_generic = false
+		// this is opposite to creating a table. See instantiateHiveTable()
+
+		if (!properties.containsKey(CatalogConfig.IS_GENERIC)) {
+			// must be a hive table
+			properties.put(CatalogConfig.IS_GENERIC, String.valueOf(false));
+		} else {
+			boolean isGeneric = Boolean.valueOf(properties.get(CatalogConfig.IS_GENERIC));
+
+			if (isGeneric) {
+				properties = retrieveFlinkProperties(properties);
+			}
 		}
+
 		String comment = properties.remove(HiveCatalogConfig.COMMENT);
 
 		// Table schema
@@ -555,11 +566,24 @@ public class HiveCatalog extends AbstractCatalog {
 			properties.put(HiveCatalogConfig.COMMENT, table.getComment());
 		}
 
-		boolean isGeneric = Boolean.valueOf(properties.get(CatalogConfig.IS_GENERIC));
+		// Make sure there's no null in properties
+		properties = cleanNullProperties(properties);
 
-		if (isGeneric) {
+		// When creating a table, A hive table needs explicitly have a key is_generic = false
+		// otherwise, this is a generic table if 1) the key is missing 2) is_generic = true
+		// this is opposite to reading a table and instantiating a CatalogTable. See instantiateCatalogTable()
+		if (!properties.containsKey(CatalogConfig.IS_GENERIC)) {
+			// must be a generic catalog
+			properties.put(CatalogConfig.IS_GENERIC, String.valueOf(true));
 			properties = maskFlinkProperties(properties);
+		} else {
+			boolean isGeneric = Boolean.valueOf(properties.get(CatalogConfig.IS_GENERIC));
+
+			if (isGeneric) {
+				properties = maskFlinkProperties(properties);
+			}
 		}
+
 		// Table properties
 		hiveTable.setParameters(properties);
 
@@ -602,6 +626,12 @@ public class HiveCatalog extends AbstractCatalog {
 		return hiveTable;
 	}
 
+	private static Map<String, String> cleanNullProperties(Map<String, String> properties) {
+		return properties.entrySet().stream()
+			.filter(e -> e.getKey() != null && e.getValue() != null)
+			.collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue()));
+	}
+
 	private static void setStorageFormat(StorageDescriptor sd, Map<String, String> properties) {
 		// TODO: allow user to specify storage format. Simply use text format for now
 		String storageFormatName = DEFAULT_HIVE_TABLE_STORAGE_FORMAT;
diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSinkTest.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSinkTest.java
index 48cb741..0bf9504 100644
--- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSinkTest.java
+++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSinkTest.java
@@ -31,6 +31,7 @@ import org.apache.flink.table.catalog.CatalogPartitionSpec;
 import org.apache.flink.table.catalog.CatalogTable;
 import org.apache.flink.table.catalog.CatalogTableImpl;
 import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.catalog.config.CatalogConfig;
 import org.apache.flink.table.catalog.hive.HiveCatalog;
 import org.apache.flink.table.catalog.hive.HiveTestUtils;
 import org.apache.flink.table.sources.InputFormatTableSource;
@@ -281,7 +282,7 @@ public class HiveTableSinkTest {
 	}
 
 	private RowTypeInfo createDestTable(String dbName, String tblName, TableSchema tableSchema, int numPartCols) throws Exception {
-		CatalogTable catalogTable = createCatalogTable(tableSchema, numPartCols);
+		CatalogTable catalogTable = createHiveCatalogTable(tableSchema, numPartCols);
 		hiveCatalog.createTable(new ObjectPath(dbName, tblName), catalogTable, false);
 		return new RowTypeInfo(tableSchema.getFieldTypes(), tableSchema.getFieldNames());
 	}
@@ -297,13 +298,26 @@ public class HiveTableSinkTest {
 		return createDestTable(dbName, tblName, builder.build(), numPartCols);
 	}
 
-	private CatalogTable createCatalogTable(TableSchema tableSchema, int numPartCols) {
+	private CatalogTable createHiveCatalogTable(TableSchema tableSchema, int numPartCols) {
 		if (numPartCols == 0) {
-			return new CatalogTableImpl(tableSchema, new HashMap<>(), "");
+			return new CatalogTableImpl(
+				tableSchema,
+				new HashMap<String, String>() {{
+					// creating a hive table needs explicit is_generic=false flag
+					put(CatalogConfig.IS_GENERIC, String.valueOf(false));
+				}},
+				"");
 		}
 		String[] partCols = new String[numPartCols];
 		System.arraycopy(tableSchema.getFieldNames(), tableSchema.getFieldNames().length - numPartCols, partCols, 0, numPartCols);
-		return new CatalogTableImpl(tableSchema, Arrays.asList(partCols), new HashMap<>(), "");
+		return new CatalogTableImpl(
+			tableSchema,
+			Arrays.asList(partCols),
+			new HashMap<String, String>() {{
+				// creating a hive table needs explicit is_generic=false flag
+				put(CatalogConfig.IS_GENERIC, String.valueOf(false));
+			}},
+			"");
 	}
 
 	private void verifyWrittenData(List<Row> expected, List<String> results) throws Exception {
diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogGenericMetadataTest.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogGenericMetadataTest.java
index b9a52b6..4ede982 100644
--- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogGenericMetadataTest.java
+++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogGenericMetadataTest.java
@@ -26,7 +26,7 @@ import org.junit.Test;
 /**
  * Test for HiveCatalog on generic metadata.
  */
-public class HiveCatalogGenericMetadataTest extends HiveCatalogTestBase {
+public class HiveCatalogGenericMetadataTest extends HiveCatalogMetadataTestBase {
 
 	@BeforeClass
 	public static void init() {
diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogHiveMetadataTest.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogHiveMetadataTest.java
index 0fe10d7..ed3c7a2 100644
--- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogHiveMetadataTest.java
+++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogHiveMetadataTest.java
@@ -46,7 +46,7 @@ import static org.junit.Assert.assertFalse;
 /**
  * Test for HiveCatalog on Hive metadata.
  */
-public class HiveCatalogHiveMetadataTest extends HiveCatalogTestBase {
+public class HiveCatalogHiveMetadataTest extends HiveCatalogMetadataTestBase {
 
 	@BeforeClass
 	public static void init() {
diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogITCase.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogITCase.java
index 0898ac3..9e28001 100644
--- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogITCase.java
+++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogITCase.java
@@ -28,6 +28,7 @@ import org.apache.flink.table.api.java.BatchTableEnvironment;
 import org.apache.flink.table.catalog.CatalogTable;
 import org.apache.flink.table.catalog.CatalogTableBuilder;
 import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.catalog.exceptions.TableNotExistException;
 import org.apache.flink.table.descriptors.FileSystem;
 import org.apache.flink.table.descriptors.FormatDescriptor;
 import org.apache.flink.table.descriptors.OldCsv;
@@ -70,8 +71,12 @@ public class HiveCatalogITCase {
 	private static HiveCatalog hiveCatalog;
 	private static HiveConf hiveConf;
 
-	private String sourceTableName = "csv_source";
-	private String sinkTableName = "csv_sink";
+	private static String sourceTableName = "csv_source";
+	private static String sinkTableName = "csv_sink";
+	private static ObjectPath source = new ObjectPath("default", sourceTableName);
+	private static ObjectPath sink = new ObjectPath("default", sinkTableName);
+
+	private String testFilePath = this.getClass().getResource("/csv/test.csv").getPath();
 
 	@BeforeClass
 	public static void createCatalog() {
@@ -81,14 +86,49 @@ public class HiveCatalogITCase {
 	}
 
 	@AfterClass
-	public static void closeCatalog() {
+	public static void closeCatalog() throws TableNotExistException {
+		hiveCatalog.dropTable(source, true);
+		hiveCatalog.dropTable(sink, true);
+
 		if (hiveCatalog != null) {
 			hiveCatalog.close();
 		}
 	}
 
 	@Test
-	public void testGenericTable() throws Exception {
+	public void testCsvTableViaDDL() throws Exception {
+		ExecutionEnvironment execEnv = ExecutionEnvironment.createLocalEnvironment(1);
+		BatchTableEnvironment tableEnv = BatchTableEnvironment.create(execEnv);
+
+		tableEnv.registerCatalog("myhive", hiveCatalog);
+		tableEnv.useCatalog("myhive");
+
+		tableEnv.sqlUpdate("create table myhive.`default`.test2 (name String, age Int) with (\n" +
+			"   'connector.type' = 'filesystem',\n" +
+			"   'connector.path' = 'file://" + testFilePath + "',\n" +
+			"   'format.type' = 'csv',\n" +
+			"   'format.fields.0.name' = 'name',\n" +
+			"   'format.fields.0.type' = 'STRING',\n" +
+			"   'format.fields.1.name' = 'age',\n" +
+			"   'format.fields.1.type' = 'INT'\n" +
+			")");
+
+		Table t = tableEnv.sqlQuery("select * from test2");
+
+		List<Row> result = tableEnv.toDataSet(t, Row.class).collect();
+
+		// assert query result
+		assertEquals(
+			Arrays.asList(
+				Row.of("1", 1),
+				Row.of("2", 2),
+				Row.of("3", 3)),
+			result
+		);
+	}
+
+	@Test
+	public void testCsvTableViaAPI() throws Exception {
 		ExecutionEnvironment execEnv = ExecutionEnvironment.createLocalEnvironment(1);
 		BatchTableEnvironment tableEnv = BatchTableEnvironment.create(execEnv);
 
@@ -105,7 +145,7 @@ public class HiveCatalogITCase {
 
 		CatalogTable source =
 			new CatalogTableBuilder(
-				new FileSystem().path(this.getClass().getResource("/csv/test.csv").getPath()),
+				new FileSystem().path(testFilePath),
 				schema)
 			.withFormat(format)
 			.inAppendMode()
diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogTestBase.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogMetadataTestBase.java
similarity index 95%
rename from flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogTestBase.java
rename to flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogMetadataTestBase.java
index 232c250..040131d 100644
--- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogTestBase.java
+++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/catalog/hive/HiveCatalogMetadataTestBase.java
@@ -26,7 +26,7 @@ import org.junit.Test;
 /**
  * Base class for testing HiveCatalog.
  */
-public abstract class HiveCatalogTestBase extends CatalogTestBase {
+public abstract class HiveCatalogMetadataTestBase extends CatalogTestBase {
 
 	// ------ functions ------