You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2023/11/14 06:16:36 UTC

(incubator-paimon) branch master updated: [Bug] Fix alter table column do not work for the non-physical column (#2297)

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new 6d31b20ad [Bug] Fix alter table column do not work for the non-physical column (#2297)
6d31b20ad is described below

commit 6d31b20adf3b7f21b307815dbee41a34ee207446
Author: Aitozi <10...@qq.com>
AuthorDate: Tue Nov 14 14:16:32 2023 +0800

    [Bug] Fix alter table column do not work for the non-physical column (#2297)
---
 .../java/org/apache/paimon/flink/FlinkCatalog.java | 132 +++++++++++++++------
 .../flink/utils/FlinkCatalogPropertiesUtil.java    |  83 ++++++++++++-
 .../apache/paimon/flink/CatalogTableITCase.java    |  20 ++++
 3 files changed, 195 insertions(+), 40 deletions(-)

diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java
index 8606f8441..415480fe6 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java
@@ -23,6 +23,7 @@ import org.apache.paimon.catalog.Catalog;
 import org.apache.paimon.catalog.Identifier;
 import org.apache.paimon.data.BinaryRow;
 import org.apache.paimon.flink.procedure.ProcedureUtil;
+import org.apache.paimon.flink.utils.FlinkCatalogPropertiesUtil;
 import org.apache.paimon.options.Options;
 import org.apache.paimon.schema.Schema;
 import org.apache.paimon.schema.SchemaChange;
@@ -49,6 +50,7 @@ import org.apache.flink.table.catalog.CatalogPartitionSpec;
 import org.apache.flink.table.catalog.CatalogTable;
 import org.apache.flink.table.catalog.CatalogTableImpl;
 import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.catalog.ResolvedCatalogBaseTable;
 import org.apache.flink.table.catalog.TableChange;
 import org.apache.flink.table.catalog.TableChange.AddColumn;
 import org.apache.flink.table.catalog.TableChange.AddWatermark;
@@ -352,27 +354,33 @@ public class FlinkCatalog extends AbstractCatalog {
         return fromCatalogTable(catalogTable);
     }
 
-    private List<SchemaChange> toSchemaChange(TableChange change) {
+    private List<SchemaChange> toSchemaChange(
+            TableChange change, Map<String, Integer> oldTableNonPhysicalColumnIndex) {
         List<SchemaChange> schemaChanges = new ArrayList<>();
         if (change instanceof AddColumn) {
-            AddColumn add = (AddColumn) change;
-            String comment = add.getColumn().getComment().orElse(null);
-            SchemaChange.Move move = getMove(add.getPosition(), add.getColumn().getName());
-            schemaChanges.add(
-                    SchemaChange.addColumn(
-                            add.getColumn().getName(),
-                            LogicalTypeConversion.toDataType(
-                                    add.getColumn().getDataType().getLogicalType()),
-                            comment,
-                            move));
+            if (((AddColumn) change).getColumn().isPhysical()) {
+                AddColumn add = (AddColumn) change;
+                String comment = add.getColumn().getComment().orElse(null);
+                SchemaChange.Move move = getMove(add.getPosition(), add.getColumn().getName());
+                schemaChanges.add(
+                        SchemaChange.addColumn(
+                                add.getColumn().getName(),
+                                LogicalTypeConversion.toDataType(
+                                        add.getColumn().getDataType().getLogicalType()),
+                                comment,
+                                move));
+            }
             return schemaChanges;
         } else if (change instanceof AddWatermark) {
             AddWatermark add = (AddWatermark) change;
             setWatermarkOptions(add.getWatermark(), schemaChanges);
             return schemaChanges;
         } else if (change instanceof DropColumn) {
-            DropColumn drop = (DropColumn) change;
-            schemaChanges.add(SchemaChange.dropColumn(drop.getColumnName()));
+            if (!oldTableNonPhysicalColumnIndex.containsKey(
+                    ((DropColumn) change).getColumnName())) {
+                DropColumn drop = (DropColumn) change;
+                schemaChanges.add(SchemaChange.dropColumn(drop.getColumnName()));
+            }
             return schemaChanges;
         } else if (change instanceof DropWatermark) {
             String watermarkPrefix = compoundKey(SCHEMA, WATERMARK, 0);
@@ -386,36 +394,48 @@ public class FlinkCatalog extends AbstractCatalog {
                             compoundKey(watermarkPrefix, WATERMARK_STRATEGY_DATA_TYPE)));
             return schemaChanges;
         } else if (change instanceof ModifyColumnName) {
-            ModifyColumnName modify = (ModifyColumnName) change;
-            schemaChanges.add(
-                    SchemaChange.renameColumn(
-                            modify.getOldColumnName(), modify.getNewColumnName()));
+            if (!oldTableNonPhysicalColumnIndex.containsKey(
+                    ((ModifyColumnName) change).getOldColumnName())) {
+                ModifyColumnName modify = (ModifyColumnName) change;
+                schemaChanges.add(
+                        SchemaChange.renameColumn(
+                                modify.getOldColumnName(), modify.getNewColumnName()));
+            }
             return schemaChanges;
         } else if (change instanceof ModifyPhysicalColumnType) {
-            ModifyPhysicalColumnType modify = (ModifyPhysicalColumnType) change;
-            LogicalType newColumnType = modify.getNewType().getLogicalType();
-            LogicalType oldColumnType = modify.getOldColumn().getDataType().getLogicalType();
-            if (newColumnType.isNullable() != oldColumnType.isNullable()) {
+            if (!oldTableNonPhysicalColumnIndex.containsKey(
+                    ((ModifyPhysicalColumnType) change).getOldColumn().getName())) {
+                ModifyPhysicalColumnType modify = (ModifyPhysicalColumnType) change;
+                LogicalType newColumnType = modify.getNewType().getLogicalType();
+                LogicalType oldColumnType = modify.getOldColumn().getDataType().getLogicalType();
+                if (newColumnType.isNullable() != oldColumnType.isNullable()) {
+                    schemaChanges.add(
+                            SchemaChange.updateColumnNullability(
+                                    modify.getNewColumn().getName(), newColumnType.isNullable()));
+                }
                 schemaChanges.add(
-                        SchemaChange.updateColumnNullability(
-                                modify.getNewColumn().getName(), newColumnType.isNullable()));
+                        SchemaChange.updateColumnType(
+                                modify.getOldColumn().getName(),
+                                LogicalTypeConversion.toDataType(newColumnType)));
             }
-            schemaChanges.add(
-                    SchemaChange.updateColumnType(
-                            modify.getOldColumn().getName(),
-                            LogicalTypeConversion.toDataType(newColumnType)));
             return schemaChanges;
         } else if (change instanceof ModifyColumnPosition) {
-            ModifyColumnPosition modify = (ModifyColumnPosition) change;
-            SchemaChange.Move move =
-                    getMove(modify.getNewPosition(), modify.getNewColumn().getName());
-            schemaChanges.add(SchemaChange.updateColumnPosition(move));
+            if (!oldTableNonPhysicalColumnIndex.containsKey(
+                    ((ModifyColumnPosition) change).getOldColumn().getName())) {
+                ModifyColumnPosition modify = (ModifyColumnPosition) change;
+                SchemaChange.Move move =
+                        getMove(modify.getNewPosition(), modify.getNewColumn().getName());
+                schemaChanges.add(SchemaChange.updateColumnPosition(move));
+            }
             return schemaChanges;
         } else if (change instanceof TableChange.ModifyColumnComment) {
-            ModifyColumnComment modify = (ModifyColumnComment) change;
-            schemaChanges.add(
-                    SchemaChange.updateColumnComment(
-                            modify.getNewColumn().getName(), modify.getNewComment()));
+            if (!oldTableNonPhysicalColumnIndex.containsKey(
+                    ((ModifyColumnComment) change).getOldColumn().getName())) {
+                ModifyColumnComment modify = (ModifyColumnComment) change;
+                schemaChanges.add(
+                        SchemaChange.updateColumnComment(
+                                modify.getNewColumn().getName(), modify.getNewComment()));
+            }
             return schemaChanges;
         } else if (change instanceof ModifyWatermark) {
             ModifyWatermark modify = (ModifyWatermark) change;
@@ -499,15 +519,49 @@ public class FlinkCatalog extends AbstractCatalog {
             return;
         }
 
-        CatalogTable table = getTable(tablePath);
+        Table table;
+        try {
+            table = catalog.getTable(toIdentifier(tablePath));
+        } catch (Catalog.TableNotExistException e) {
+            throw new TableNotExistException(getName(), tablePath);
+        }
 
-        validateAlterTable(table, (CatalogTable) newTable);
+        Preconditions.checkArgument(table instanceof FileStoreTable, "Can't alter system table.");
+        validateAlterTable(toCatalogTable(table), (CatalogTable) newTable);
+        Map<String, Integer> oldTableNonPhysicalColumnIndex =
+                FlinkCatalogPropertiesUtil.nonPhysicalColumns(
+                        table.options(), table.rowType().getFieldNames());
 
         List<SchemaChange> changes = new ArrayList<>();
+
+        Map<String, String> schemaOptions =
+                FlinkCatalogPropertiesUtil.serializeNonPhysicalNewColumns(
+                        ((ResolvedCatalogBaseTable<?>) newTable).getResolvedSchema());
+        table.options()
+                .forEach(
+                        (k, v) -> {
+                            if (FlinkCatalogPropertiesUtil.isNonPhysicalColumnKey(k)) {
+                                // drop non-physical column
+                                if (!schemaOptions.containsKey(k)) {
+                                    changes.add(SchemaChange.removeOption(k));
+                                }
+                            }
+                        });
+        schemaOptions.forEach(
+                (k, v) -> {
+                    // add/modify non-physical column
+                    if (!table.options().containsKey(k) || !table.options().get(k).equals(v)) {
+                        changes.add(SchemaChange.setOption(k, v));
+                    }
+                });
         if (null != tableChanges) {
             List<SchemaChange> schemaChanges =
                     tableChanges.stream()
-                            .flatMap(tableChange -> toSchemaChange(tableChange).stream())
+                            .flatMap(
+                                    tableChange ->
+                                            toSchemaChange(
+                                                    tableChange, oldTableNonPhysicalColumnIndex)
+                                                    .stream())
                             .collect(Collectors.toList());
             changes.addAll(schemaChanges);
         }
@@ -517,7 +571,7 @@ public class FlinkCatalog extends AbstractCatalog {
         } catch (Catalog.TableNotExistException
                 | Catalog.ColumnAlreadyExistException
                 | Catalog.ColumnNotExistException e) {
-            throw new TableNotExistException(getName(), tablePath);
+            throw new CatalogException(e);
         }
     }
 
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/FlinkCatalogPropertiesUtil.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/FlinkCatalogPropertiesUtil.java
index 011a1a4e0..fb019c6e4 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/FlinkCatalogPropertiesUtil.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/FlinkCatalogPropertiesUtil.java
@@ -18,8 +18,12 @@
 
 package org.apache.paimon.flink.utils;
 
+import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableSet;
+
 import org.apache.flink.table.api.TableColumn;
 import org.apache.flink.table.api.WatermarkSpec;
+import org.apache.flink.table.catalog.Column;
+import org.apache.flink.table.catalog.ResolvedSchema;
 import org.apache.flink.table.types.DataType;
 import org.apache.flink.table.types.logical.utils.LogicalTypeParser;
 import org.apache.flink.table.types.utils.TypeConversions;
@@ -27,6 +31,7 @@ import org.apache.flink.table.types.utils.TypeConversions;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -73,6 +78,38 @@ public class FlinkCatalogPropertiesUtil {
         return serialized;
     }
 
+    /** Serialize non-physical columns of new api. */
+    public static Map<String, String> serializeNonPhysicalNewColumns(ResolvedSchema schema) {
+        List<Column> nonPhysicalColumns =
+                schema.getColumns().stream()
+                        .filter(k -> !k.isPhysical())
+                        .collect(Collectors.toList());
+        Map<String, String> serialized = new HashMap<>();
+        List<String> columnNames = schema.getColumnNames();
+        for (Column c : nonPhysicalColumns) {
+            int index = columnNames.indexOf(c.getName());
+            serialized.put(compoundKey(SCHEMA, index, NAME), c.getName());
+            serialized.put(
+                    compoundKey(SCHEMA, index, DATA_TYPE),
+                    c.getDataType().getLogicalType().asSerializableString());
+            if (c instanceof Column.ComputedColumn) {
+                Column.ComputedColumn computedColumn = (Column.ComputedColumn) c;
+                serialized.put(
+                        compoundKey(SCHEMA, index, EXPR),
+                        computedColumn.getExpression().asSerializableString());
+            } else {
+                Column.MetadataColumn metadataColumn = (Column.MetadataColumn) c;
+                serialized.put(
+                        compoundKey(SCHEMA, index, METADATA),
+                        metadataColumn.getMetadataKey().orElse(metadataColumn.getName()));
+                serialized.put(
+                        compoundKey(SCHEMA, index, VIRTUAL),
+                        Boolean.toString(metadataColumn.isVirtual()));
+            }
+        }
+        return serialized;
+    }
+
     public static Map<String, String> serializeWatermarkSpec(WatermarkSpec watermarkSpec) {
         Map<String, String> serializedWatermarkSpec = new HashMap<>();
         String watermarkPrefix = compoundKey(SCHEMA, WATERMARK, 0);
@@ -89,7 +126,21 @@ public class FlinkCatalogPropertiesUtil {
         return serializedWatermarkSpec;
     }
 
-    private static final Pattern SCHEMA_COLUMN_NAME_SUFFIX = Pattern.compile("\\d+\\.name");
+    private static final Pattern SCHEMA_COLUMN_NAME_SUFFIX = Pattern.compile("\\d+\\." + NAME);
+    private static final Pattern SCHEMA_COLUMN_METADATA_SUFFIX =
+            Pattern.compile("\\d+\\." + METADATA);
+    private static final Pattern SCHEMA_COLUMN_EXPR_SUFFIX = Pattern.compile("\\d+\\." + EXPR);
+    private static final Pattern SCHEMA_COLUMN_DATATYPE_SUFFIX =
+            Pattern.compile("\\d+\\." + DATA_TYPE);
+    private static final Pattern SCHEMA_COLUMN_VIRTUAL_SUFFIX =
+            Pattern.compile("\\d+\\." + VIRTUAL);
+    private static final Set<Pattern> NON_PHYSICAL_KEY_PATTERNS =
+            ImmutableSet.of(
+                    SCHEMA_COLUMN_NAME_SUFFIX,
+                    SCHEMA_COLUMN_METADATA_SUFFIX,
+                    SCHEMA_COLUMN_EXPR_SUFFIX,
+                    SCHEMA_COLUMN_DATATYPE_SUFFIX,
+                    SCHEMA_COLUMN_VIRTUAL_SUFFIX);
 
     public static int nonPhysicalColumnsCount(
             Map<String, String> tableOptions, List<String> physicalColumns) {
@@ -103,6 +154,36 @@ public class FlinkCatalogPropertiesUtil {
         return count;
     }
 
+    public static boolean isNonPhysicalColumnKey(String key) {
+        if (!key.startsWith(SCHEMA)) {
+            return false;
+        }
+        String suffix = key.substring(SCHEMA.length() + 1);
+        for (Pattern pattern : NON_PHYSICAL_KEY_PATTERNS) {
+            if (pattern.matcher(suffix).matches()) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public static Map<String, Integer> nonPhysicalColumns(
+            Map<String, String> tableOptions, List<String> physicalColumns) {
+        Map<String, Integer> nonPhysicalColumnIndex = new HashMap<>();
+        for (Map.Entry<String, String> entry : tableOptions.entrySet()) {
+            if (isColumnNameKey(entry.getKey()) && !physicalColumns.contains(entry.getValue())) {
+                String key = entry.getKey();
+                int index =
+                        Integer.parseInt(
+                                key.substring(
+                                        SCHEMA.length() + 1,
+                                        key.indexOf(".", SCHEMA.length() + 1)));
+                nonPhysicalColumnIndex.put(entry.getValue(), index);
+            }
+        }
+        return nonPhysicalColumnIndex;
+    }
+
     private static boolean isColumnNameKey(String key) {
         return key.startsWith(SCHEMA)
                 && SCHEMA_COLUMN_NAME_SUFFIX.matcher(key.substring(SCHEMA.length() + 1)).matches();
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java
index d5590aa6b..920c6a8b3 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java
@@ -708,4 +708,24 @@ public class CatalogTableITCase extends CatalogITCaseBase {
                                 "Cannot set streaming-read-overwrite to true when changelog producer "
                                         + "is full-compaction or lookup because it will read duplicated changes."));
     }
+
+    @Test
+    public void testAlterTableNonPhysicalColumn() {
+        sql(
+                "CREATE TABLE T (a INT,  c ROW < a INT, d INT> METADATA, b INT, ts TIMESTAMP(3), WATERMARK FOR ts AS ts)");
+        sql("ALTER TABLE T ADD e VARCHAR METADATA");
+        sql("ALTER TABLE T DROP c ");
+        sql("ALTER TABLE T RENAME e TO ee");
+        List<Row> result = sql("SHOW CREATE TABLE T");
+        assertThat(result.get(0).toString())
+                .contains(
+                        "CREATE TABLE `PAIMON`.`default`.`T` (\n"
+                                + "  `a` INT,\n"
+                                + "  `b` INT,\n"
+                                + "  `ts` TIMESTAMP(3),\n"
+                                + "  `ee` VARCHAR(2147483647) METADATA,\n"
+                                + "  WATERMARK FOR `ts` AS `ts`\n"
+                                + ") ")
+                .doesNotContain("schema");
+    }
 }