You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by st...@apache.org on 2024/03/05 02:29:56 UTC

(iceberg) branch main updated: Flink: Supports specifying comment for iceberg fields in create table and addcolumn syntax using flinksql (#9606)

This is an automated email from the ASF dual-hosted git repository.

stevenwu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/main by this push:
     new 05f99b6581 Flink: Supports specifying comment for iceberg fields in create table and addcolumn syntax using flinksql (#9606)
05f99b6581 is described below

commit 05f99b658189e277bbf32a814ee489cf59add3fa
Author: big face cat <73...@qq.com>
AuthorDate: Tue Mar 5 10:29:50 2024 +0800

    Flink: Supports specifying comment for iceberg fields in create table and addcolumn syntax using flinksql (#9606)
    
    Co-authored-by: huyuanfeng <hu...@huya.com>
---
 .../org/apache/iceberg/flink/FlinkCatalog.java     | 10 +--
 .../iceberg/flink/FlinkDynamicTableFactory.java    | 23 ++++---
 .../org/apache/iceberg/flink/FlinkSchemaUtil.java  | 77 +++++++++++++++++-----
 .../iceberg/flink/util/FlinkAlterTableUtil.java    |  6 +-
 .../iceberg/flink/TestFlinkCatalogTable.java       | 18 ++++-
 5 files changed, 95 insertions(+), 39 deletions(-)

diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java
index f022c8abcb..86295d78cc 100644
--- a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java
@@ -38,6 +38,7 @@ import org.apache.flink.table.catalog.CatalogPartitionSpec;
 import org.apache.flink.table.catalog.CatalogTable;
 import org.apache.flink.table.catalog.CatalogTableImpl;
 import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.catalog.ResolvedCatalogTable;
 import org.apache.flink.table.catalog.TableChange;
 import org.apache.flink.table.catalog.exceptions.CatalogException;
 import org.apache.flink.table.catalog.exceptions.DatabaseAlreadyExistException;
@@ -390,17 +391,16 @@ public class FlinkCatalog extends AbstractCatalog {
               + "an iceberg catalog, Please create table with 'connector'='iceberg' property in a non-iceberg catalog or "
               + "create table without 'connector'='iceberg' related properties in an iceberg table.");
     }
-
-    createIcebergTable(tablePath, table, ignoreIfExists);
+    Preconditions.checkArgument(table instanceof ResolvedCatalogTable, "table should be resolved");
+    createIcebergTable(tablePath, (ResolvedCatalogTable) table, ignoreIfExists);
   }
 
-  void createIcebergTable(ObjectPath tablePath, CatalogBaseTable table, boolean ignoreIfExists)
+  void createIcebergTable(ObjectPath tablePath, ResolvedCatalogTable table, boolean ignoreIfExists)
       throws CatalogException, TableAlreadyExistException {
     validateFlinkTable(table);
 
-    Schema icebergSchema = FlinkSchemaUtil.convert(table.getSchema());
+    Schema icebergSchema = FlinkSchemaUtil.convert(table.getResolvedSchema());
     PartitionSpec spec = toPartitionSpec(((CatalogTable) table).getPartitionKeys(), icebergSchema);
-
     ImmutableMap.Builder<String, String> properties = ImmutableMap.builder();
     String location = null;
     for (Map.Entry<String, String> entry : table.getOptions().entrySet()) {
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java
index 8e1f420b72..b7f1be4b93 100644
--- a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java
@@ -24,11 +24,10 @@ import org.apache.flink.configuration.ConfigOption;
 import org.apache.flink.configuration.ConfigOptions;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.table.api.TableSchema;
-import org.apache.flink.table.catalog.CatalogBaseTable;
 import org.apache.flink.table.catalog.CatalogDatabaseImpl;
-import org.apache.flink.table.catalog.CatalogTable;
 import org.apache.flink.table.catalog.ObjectIdentifier;
 import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.catalog.ResolvedCatalogTable;
 import org.apache.flink.table.catalog.exceptions.DatabaseAlreadyExistException;
 import org.apache.flink.table.catalog.exceptions.TableAlreadyExistException;
 import org.apache.flink.table.connector.sink.DynamicTableSink;
@@ -84,9 +83,9 @@ public class FlinkDynamicTableFactory
   @Override
   public DynamicTableSource createDynamicTableSource(Context context) {
     ObjectIdentifier objectIdentifier = context.getObjectIdentifier();
-    CatalogTable catalogTable = context.getCatalogTable();
-    Map<String, String> tableProps = catalogTable.getOptions();
-    TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema(catalogTable.getSchema());
+    ResolvedCatalogTable resolvedCatalogTable = context.getCatalogTable();
+    Map<String, String> tableProps = resolvedCatalogTable.getOptions();
+    TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema(resolvedCatalogTable.getSchema());
 
     TableLoader tableLoader;
     if (catalog != null) {
@@ -94,7 +93,7 @@ public class FlinkDynamicTableFactory
     } else {
       tableLoader =
           createTableLoader(
-              catalogTable,
+              resolvedCatalogTable,
               tableProps,
               objectIdentifier.getDatabaseName(),
               objectIdentifier.getObjectName());
@@ -106,9 +105,9 @@ public class FlinkDynamicTableFactory
   @Override
   public DynamicTableSink createDynamicTableSink(Context context) {
     ObjectIdentifier objectIdentifier = context.getObjectIdentifier();
-    CatalogTable catalogTable = context.getCatalogTable();
-    Map<String, String> writeProps = catalogTable.getOptions();
-    TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema(catalogTable.getSchema());
+    ResolvedCatalogTable resolvedCatalogTable = context.getCatalogTable();
+    Map<String, String> writeProps = resolvedCatalogTable.getOptions();
+    TableSchema tableSchema = TableSchemaUtils.getPhysicalSchema(resolvedCatalogTable.getSchema());
 
     TableLoader tableLoader;
     if (catalog != null) {
@@ -116,7 +115,7 @@ public class FlinkDynamicTableFactory
     } else {
       tableLoader =
           createTableLoader(
-              catalogTable,
+              resolvedCatalogTable,
               writeProps,
               objectIdentifier.getDatabaseName(),
               objectIdentifier.getObjectName());
@@ -147,7 +146,7 @@ public class FlinkDynamicTableFactory
   }
 
   private static TableLoader createTableLoader(
-      CatalogBaseTable catalogBaseTable,
+      ResolvedCatalogTable resolvedCatalogTable,
       Map<String, String> tableProps,
       String databaseName,
       String tableName) {
@@ -187,7 +186,7 @@ public class FlinkDynamicTableFactory
     // Create table if not exists in the external catalog.
     if (!flinkCatalog.tableExists(objectPath)) {
       try {
-        flinkCatalog.createIcebergTable(objectPath, catalogBaseTable, true);
+        flinkCatalog.createIcebergTable(objectPath, resolvedCatalogTable, true);
       } catch (TableAlreadyExistException e) {
         throw new AlreadyExistsException(
             e,
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkSchemaUtil.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkSchemaUtil.java
index a6b53879ad..4790dc85bf 100644
--- a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkSchemaUtil.java
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/FlinkSchemaUtil.java
@@ -20,7 +20,10 @@ package org.apache.iceberg.flink;
 
 import java.util.List;
 import java.util.Set;
+import org.apache.flink.table.api.DataTypes;
 import org.apache.flink.table.api.TableSchema;
+import org.apache.flink.table.catalog.Column;
+import org.apache.flink.table.catalog.ResolvedSchema;
 import org.apache.flink.table.types.logical.LogicalType;
 import org.apache.flink.table.types.logical.RowType;
 import org.apache.flink.table.types.utils.TypeConversions;
@@ -55,35 +58,69 @@ public class FlinkSchemaUtil {
 
   private FlinkSchemaUtil() {}
 
-  /** Convert the flink table schema to apache iceberg schema. */
+  /** @deprecated Use {@link #convert(ResolvedSchema)} instead. */
+  @Deprecated
   public static Schema convert(TableSchema schema) {
     LogicalType schemaType = schema.toRowDataType().getLogicalType();
     Preconditions.checkArgument(
-        schemaType instanceof RowType, "Schema logical type should be RowType.");
+        schemaType instanceof RowType, "Schema logical type should be row type.");
 
     RowType root = (RowType) schemaType;
     Type converted = root.accept(new FlinkTypeToType(root));
 
-    Schema iSchema = new Schema(converted.asStructType().fields());
-    return freshIdentifierFieldIds(iSchema, schema);
+    Schema icebergSchema = new Schema(converted.asStructType().fields());
+    if (schema.getPrimaryKey().isPresent()) {
+      return freshIdentifierFieldIds(icebergSchema, schema.getPrimaryKey().get().getColumns());
+    } else {
+      return icebergSchema;
+    }
+  }
+
+  /** Convert the flink table schema to apache iceberg schema with column comment. */
+  public static Schema convert(ResolvedSchema flinkSchema) {
+    List<Column> tableColumns = flinkSchema.getColumns();
+    // copy from org.apache.flink.table.api.Schema#toRowDataType
+    DataTypes.Field[] fields =
+        tableColumns.stream()
+            .map(
+                column -> {
+                  if (column.getComment().isPresent()) {
+                    return DataTypes.FIELD(
+                        column.getName(), column.getDataType(), column.getComment().get());
+                  } else {
+                    return DataTypes.FIELD(column.getName(), column.getDataType());
+                  }
+                })
+            .toArray(DataTypes.Field[]::new);
+
+    LogicalType schemaType = DataTypes.ROW(fields).notNull().getLogicalType();
+    Preconditions.checkArgument(
+        schemaType instanceof RowType, "Schema logical type should be row type.");
+
+    RowType root = (RowType) schemaType;
+    Type converted = root.accept(new FlinkTypeToType(root));
+    Schema icebergSchema = new Schema(converted.asStructType().fields());
+    if (flinkSchema.getPrimaryKey().isPresent()) {
+      return freshIdentifierFieldIds(icebergSchema, flinkSchema.getPrimaryKey().get().getColumns());
+    } else {
+      return icebergSchema;
+    }
   }
 
-  private static Schema freshIdentifierFieldIds(Schema iSchema, TableSchema schema) {
+  private static Schema freshIdentifierFieldIds(Schema icebergSchema, List<String> primaryKeys) {
     // Locate the identifier field id list.
     Set<Integer> identifierFieldIds = Sets.newHashSet();
-    if (schema.getPrimaryKey().isPresent()) {
-      for (String column : schema.getPrimaryKey().get().getColumns()) {
-        Types.NestedField field = iSchema.findField(column);
-        Preconditions.checkNotNull(
-            field,
-            "Cannot find field ID for the primary key column %s in schema %s",
-            column,
-            iSchema);
-        identifierFieldIds.add(field.fieldId());
-      }
+    for (String primaryKey : primaryKeys) {
+      Types.NestedField field = icebergSchema.findField(primaryKey);
+      Preconditions.checkNotNull(
+          field,
+          "Cannot find field ID for the primary key column %s in schema %s",
+          primaryKey,
+          icebergSchema);
+      identifierFieldIds.add(field.fieldId());
     }
-
-    return new Schema(iSchema.schemaId(), iSchema.asStruct().fields(), identifierFieldIds);
+    return new Schema(
+        icebergSchema.schemaId(), icebergSchema.asStruct().fields(), identifierFieldIds);
   }
 
   /**
@@ -109,7 +146,11 @@ public class FlinkSchemaUtil {
 
     // fix types that can't be represented in Flink (UUID)
     Schema fixedSchema = FlinkFixupTypes.fixup(schema, baseSchema);
-    return freshIdentifierFieldIds(fixedSchema, flinkSchema);
+    if (flinkSchema.getPrimaryKey().isPresent()) {
+      return freshIdentifierFieldIds(fixedSchema, flinkSchema.getPrimaryKey().get().getColumns());
+    } else {
+      return fixedSchema;
+    }
   }
 
   /**
diff --git a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/util/FlinkAlterTableUtil.java b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/util/FlinkAlterTableUtil.java
index f0b9bf64fb..2bbc9cf208 100644
--- a/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/util/FlinkAlterTableUtil.java
+++ b/flink/v1.18/flink/src/main/java/org/apache/iceberg/flink/util/FlinkAlterTableUtil.java
@@ -132,9 +132,11 @@ public class FlinkAlterTableUtil {
             flinkColumn.getName());
         Type icebergType = FlinkSchemaUtil.convert(flinkColumn.getDataType().getLogicalType());
         if (flinkColumn.getDataType().getLogicalType().isNullable()) {
-          pendingUpdate.addColumn(flinkColumn.getName(), icebergType);
+          pendingUpdate.addColumn(
+              flinkColumn.getName(), icebergType, flinkColumn.getComment().orElse(null));
         } else {
-          pendingUpdate.addRequiredColumn(flinkColumn.getName(), icebergType);
+          pendingUpdate.addRequiredColumn(
+              flinkColumn.getName(), icebergType, flinkColumn.getComment().orElse(null));
         }
       } else if (change instanceof TableChange.ModifyColumn) {
         TableChange.ModifyColumn modifyColumn = (TableChange.ModifyColumn) change;
diff --git a/flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java b/flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java
index ef0802d869..eaa92e32c4 100644
--- a/flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java
+++ b/flink/v1.18/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java
@@ -226,6 +226,19 @@ public class TestFlinkCatalogTable extends CatalogTestBase {
     assertThat(catalogTable.getPartitionKeys()).isEqualTo(Collections.singletonList("dt"));
   }
 
+  @TestTemplate
+  public void testCreateTableWithColumnComment() {
+    sql("CREATE TABLE tl(id BIGINT COMMENT 'comment - id', data STRING COMMENT 'comment - data')");
+
+    Table table = table("tl");
+    assertThat(table.schema().asStruct())
+        .isEqualTo(
+            new Schema(
+                    Types.NestedField.optional(1, "id", Types.LongType.get(), "comment - id"),
+                    Types.NestedField.optional(2, "data", Types.StringType.get(), "comment - data"))
+                .asStruct());
+  }
+
   @TestTemplate
   public void testCreateTableWithFormatV2ThroughTableProperty() throws Exception {
     sql("CREATE TABLE tl(id BIGINT) WITH ('format-version'='2')");
@@ -316,14 +329,15 @@ public class TestFlinkCatalogTable extends CatalogTestBase {
                     Types.NestedField.optional(2, "dt", Types.StringType.get()))
                 .asStruct());
     // Add multiple columns
-    sql("ALTER TABLE tl ADD (col1 STRING, col2 BIGINT)");
+    sql("ALTER TABLE tl ADD (col1 STRING COMMENT 'comment for col1', col2 BIGINT)");
     Schema schemaAfter2 = table("tl").schema();
     assertThat(schemaAfter2.asStruct())
         .isEqualTo(
             new Schema(
                     Types.NestedField.optional(1, "id", Types.LongType.get()),
                     Types.NestedField.optional(2, "dt", Types.StringType.get()),
-                    Types.NestedField.optional(3, "col1", Types.StringType.get()),
+                    Types.NestedField.optional(
+                        3, "col1", Types.StringType.get(), "comment for col1"),
                     Types.NestedField.optional(4, "col2", Types.LongType.get()))
                 .asStruct());
     // Adding a required field should fail because Iceberg's SchemaUpdate does not allow