You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by lz...@apache.org on 2022/11/15 06:29:47 UTC

[flink-table-store] branch master updated: [FLINK-29922] Support create external table for hive catalog

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git


The following commit(s) were added to refs/heads/master by this push:
     new feffebde [FLINK-29922] Support create external table for hive catalog
feffebde is described below

commit feffebde7c3163232636c5b22add7d311198d48e
Author: 吴祥平 <40...@qq.com>
AuthorDate: Tue Nov 15 14:29:43 2022 +0800

    [FLINK-29922] Support create external table for hive catalog
    
    This closes #357
---
 .../generated/catalog_configuration.html           |  6 +++
 .../apache/flink/table/store/CatalogOptions.java   |  7 +++
 .../apache/flink/table/store/table/TableType.java  | 52 ++++++++++++++++++++++
 .../apache/flink/table/store/hive/HiveCatalog.java |  9 +++-
 .../flink/table/store/hive/HiveCatalogITCase.java  | 24 ++++++++++
 5 files changed, 96 insertions(+), 2 deletions(-)

diff --git a/docs/layouts/shortcodes/generated/catalog_configuration.html b/docs/layouts/shortcodes/generated/catalog_configuration.html
index b3056129..57454f95 100644
--- a/docs/layouts/shortcodes/generated/catalog_configuration.html
+++ b/docs/layouts/shortcodes/generated/catalog_configuration.html
@@ -32,6 +32,12 @@
             <td>String</td>
             <td>Metastore of table store catalog, supports filesystem and hive.</td>
         </tr>
+        <tr>
+            <td><h5>table.type</h5></td>
+            <td style="word-wrap: break-word;">MANAGED_TABLE</td>
+            <td><p>Enum</p></td>
+            <td>Type of table.<br /><br />Possible values:<ul><li>"MANAGED_TABLE": Table Store owned table where the entire lifecycle of the table data is managed.</li><li>"EXTERNAL_TABLE": The table where Table Store has loose coupling with the data stored in external locations.</li></ul></td>
+        </tr>
         <tr>
             <td><h5>uri</h5></td>
             <td style="word-wrap: break-word;">(none)</td>
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/CatalogOptions.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/CatalogOptions.java
index fa8b3b01..a2c806bb 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/CatalogOptions.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/CatalogOptions.java
@@ -21,6 +21,7 @@ package org.apache.flink.table.store;
 import org.apache.flink.configuration.ConfigOption;
 import org.apache.flink.configuration.ConfigOptions;
 import org.apache.flink.table.store.file.catalog.FileSystemCatalogFactory;
+import org.apache.flink.table.store.table.TableType;
 
 import java.time.Duration;
 
@@ -49,6 +50,12 @@ public class CatalogOptions {
                     .noDefaultValue()
                     .withDescription("Uri of metastore server.");
 
+    public static final ConfigOption<TableType> TABLE_TYPE =
+            ConfigOptions.key("table.type")
+                    .enumType(TableType.class)
+                    .defaultValue(TableType.MANAGED)
+                    .withDescription("Type of table.");
+
     public static final ConfigOption<Boolean> LOCK_ENABLED =
             ConfigOptions.key("lock.enabled")
                     .booleanType()
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/TableType.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/TableType.java
new file mode 100644
index 00000000..0e984bd7
--- /dev/null
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/table/TableType.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.table;
+
+import org.apache.flink.configuration.DescribedEnum;
+import org.apache.flink.configuration.description.InlineElement;
+
+import static org.apache.flink.configuration.description.TextElement.text;
+
+/** Enum of catalog table type. */
+public enum TableType implements DescribedEnum {
+    MANAGED(
+            "MANAGED_TABLE",
+            "Table Store owned table where the entire lifecycle of the table data is managed."),
+    EXTERNAL(
+            "EXTERNAL_TABLE",
+            "The table where Table Store has loose coupling with the data stored in external locations.");
+
+    private final String value;
+    private final String description;
+
+    TableType(String value, String description) {
+        this.value = value;
+        this.description = description;
+    }
+
+    @Override
+    public String toString() {
+        return value;
+    }
+
+    @Override
+    public InlineElement getDescription() {
+        return text(description);
+    }
+}
diff --git a/flink-table-store-hive/flink-table-store-hive-catalog/src/main/java/org/apache/flink/table/store/hive/HiveCatalog.java b/flink-table-store-hive/flink-table-store-hive-catalog/src/main/java/org/apache/flink/table/store/hive/HiveCatalog.java
index aa59e2f2..59e66f00 100644
--- a/flink-table-store-hive/flink-table-store-hive-catalog/src/main/java/org/apache/flink/table/store/hive/HiveCatalog.java
+++ b/flink-table-store-hive/flink-table-store-hive-catalog/src/main/java/org/apache/flink/table/store/hive/HiveCatalog.java
@@ -28,6 +28,7 @@ import org.apache.flink.table.store.file.schema.SchemaChange;
 import org.apache.flink.table.store.file.schema.SchemaManager;
 import org.apache.flink.table.store.file.schema.TableSchema;
 import org.apache.flink.table.store.file.schema.UpdateSchema;
+import org.apache.flink.table.store.table.TableType;
 import org.apache.flink.util.StringUtils;
 
 import org.apache.hadoop.conf.Configuration;
@@ -35,7 +36,6 @@ import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
 import org.apache.hadoop.hive.metastore.IMetaStoreClient;
 import org.apache.hadoop.hive.metastore.RetryingMetaStoreClient;
-import org.apache.hadoop.hive.metastore.TableType;
 import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -55,6 +55,7 @@ import java.util.Optional;
 import java.util.stream.Collectors;
 
 import static org.apache.flink.table.store.CatalogOptions.LOCK_ENABLED;
+import static org.apache.flink.table.store.CatalogOptions.TABLE_TYPE;
 import static org.apache.flink.table.store.hive.HiveCatalogLock.acquireTimeout;
 import static org.apache.flink.table.store.hive.HiveCatalogLock.checkMaxSleep;
 
@@ -278,6 +279,7 @@ public class HiveCatalog extends AbstractCatalog {
 
     private Table newHmsTable(ObjectPath tablePath) {
         long currentTimeMillis = System.currentTimeMillis();
+        final TableType tableType = hiveConf.getEnum(TABLE_TYPE.key(), TableType.MANAGED);
         Table table =
                 new Table(
                         tablePath.getObjectName(),
@@ -292,9 +294,12 @@ public class HiveCatalog extends AbstractCatalog {
                         new HashMap<>(),
                         null,
                         null,
-                        TableType.MANAGED_TABLE.toString());
+                        tableType.toString());
         table.getParameters()
                 .put(hive_metastoreConstants.META_TABLE_STORAGE, STORAGE_HANDLER_CLASS_NAME);
+        if (TableType.EXTERNAL.equals(tableType)) {
+            table.getParameters().put("EXTERNAL", "TRUE");
+        }
         return table;
     }
 
diff --git a/flink-table-store-hive/flink-table-store-hive-catalog/src/test/java/org/apache/flink/table/store/hive/HiveCatalogITCase.java b/flink-table-store-hive/flink-table-store-hive-catalog/src/test/java/org/apache/flink/table/store/hive/HiveCatalogITCase.java
index b44878e4..27a85264 100644
--- a/flink-table-store-hive/flink-table-store-hive-catalog/src/test/java/org/apache/flink/table/store/hive/HiveCatalogITCase.java
+++ b/flink-table-store-hive/flink-table-store-hive-catalog/src/test/java/org/apache/flink/table/store/hive/HiveCatalogITCase.java
@@ -230,6 +230,30 @@ public class HiveCatalogITCase {
         }
     }
 
+    @Test
+    public void testCreateExternalTable() throws Exception {
+        tEnv.executeSql(
+                        String.join(
+                                "\n",
+                                "CREATE CATALOG my_hive_external WITH (",
+                                "  'type' = 'table-store',",
+                                "  'metastore' = 'hive',",
+                                "  'uri' = '',",
+                                "  'warehouse' = '" + path + "',",
+                                "  'lock.enabled' = 'true',",
+                                "  'table.type' = 'EXTERNAL'",
+                                ")"))
+                .await();
+        tEnv.executeSql("USE CATALOG my_hive_external").await();
+        tEnv.executeSql("USE test_db").await();
+        tEnv.executeSql("CREATE TABLE T ( a INT, b STRING ) WITH ( 'file.format' = 'avro' )")
+                .await();
+        Assert.assertTrue(
+                hiveShell
+                        .executeQuery("DESC FORMATTED t")
+                        .contains("Table Type:         \tEXTERNAL_TABLE      \tNULL"));
+    }
+
     @Test
     public void testFlinkWriteAndHiveRead() throws Exception {
         tEnv.executeSql("CREATE TABLE T ( a INT, b STRING ) WITH ( 'file.format' = 'avro' )")