You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2023/11/14 06:16:36 UTC
(incubator-paimon) branch master updated: [Bug] Fix alter table column do not work for the non-physical column (#2297)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 6d31b20ad [Bug] Fix alter table column do not work for the non-physical column (#2297)
6d31b20ad is described below
commit 6d31b20adf3b7f21b307815dbee41a34ee207446
Author: Aitozi <10...@qq.com>
AuthorDate: Tue Nov 14 14:16:32 2023 +0800
[Bug] Fix alter table column do not work for the non-physical column (#2297)
---
.../java/org/apache/paimon/flink/FlinkCatalog.java | 132 +++++++++++++++------
.../flink/utils/FlinkCatalogPropertiesUtil.java | 83 ++++++++++++-
.../apache/paimon/flink/CatalogTableITCase.java | 20 ++++
3 files changed, 195 insertions(+), 40 deletions(-)
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java
index 8606f8441..415480fe6 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java
@@ -23,6 +23,7 @@ import org.apache.paimon.catalog.Catalog;
import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.flink.procedure.ProcedureUtil;
+import org.apache.paimon.flink.utils.FlinkCatalogPropertiesUtil;
import org.apache.paimon.options.Options;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.SchemaChange;
@@ -49,6 +50,7 @@ import org.apache.flink.table.catalog.CatalogPartitionSpec;
import org.apache.flink.table.catalog.CatalogTable;
import org.apache.flink.table.catalog.CatalogTableImpl;
import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.catalog.ResolvedCatalogBaseTable;
import org.apache.flink.table.catalog.TableChange;
import org.apache.flink.table.catalog.TableChange.AddColumn;
import org.apache.flink.table.catalog.TableChange.AddWatermark;
@@ -352,27 +354,33 @@ public class FlinkCatalog extends AbstractCatalog {
return fromCatalogTable(catalogTable);
}
- private List<SchemaChange> toSchemaChange(TableChange change) {
+ private List<SchemaChange> toSchemaChange(
+ TableChange change, Map<String, Integer> oldTableNonPhysicalColumnIndex) {
List<SchemaChange> schemaChanges = new ArrayList<>();
if (change instanceof AddColumn) {
- AddColumn add = (AddColumn) change;
- String comment = add.getColumn().getComment().orElse(null);
- SchemaChange.Move move = getMove(add.getPosition(), add.getColumn().getName());
- schemaChanges.add(
- SchemaChange.addColumn(
- add.getColumn().getName(),
- LogicalTypeConversion.toDataType(
- add.getColumn().getDataType().getLogicalType()),
- comment,
- move));
+ if (((AddColumn) change).getColumn().isPhysical()) {
+ AddColumn add = (AddColumn) change;
+ String comment = add.getColumn().getComment().orElse(null);
+ SchemaChange.Move move = getMove(add.getPosition(), add.getColumn().getName());
+ schemaChanges.add(
+ SchemaChange.addColumn(
+ add.getColumn().getName(),
+ LogicalTypeConversion.toDataType(
+ add.getColumn().getDataType().getLogicalType()),
+ comment,
+ move));
+ }
return schemaChanges;
} else if (change instanceof AddWatermark) {
AddWatermark add = (AddWatermark) change;
setWatermarkOptions(add.getWatermark(), schemaChanges);
return schemaChanges;
} else if (change instanceof DropColumn) {
- DropColumn drop = (DropColumn) change;
- schemaChanges.add(SchemaChange.dropColumn(drop.getColumnName()));
+ if (!oldTableNonPhysicalColumnIndex.containsKey(
+ ((DropColumn) change).getColumnName())) {
+ DropColumn drop = (DropColumn) change;
+ schemaChanges.add(SchemaChange.dropColumn(drop.getColumnName()));
+ }
return schemaChanges;
} else if (change instanceof DropWatermark) {
String watermarkPrefix = compoundKey(SCHEMA, WATERMARK, 0);
@@ -386,36 +394,48 @@ public class FlinkCatalog extends AbstractCatalog {
compoundKey(watermarkPrefix, WATERMARK_STRATEGY_DATA_TYPE)));
return schemaChanges;
} else if (change instanceof ModifyColumnName) {
- ModifyColumnName modify = (ModifyColumnName) change;
- schemaChanges.add(
- SchemaChange.renameColumn(
- modify.getOldColumnName(), modify.getNewColumnName()));
+ if (!oldTableNonPhysicalColumnIndex.containsKey(
+ ((ModifyColumnName) change).getOldColumnName())) {
+ ModifyColumnName modify = (ModifyColumnName) change;
+ schemaChanges.add(
+ SchemaChange.renameColumn(
+ modify.getOldColumnName(), modify.getNewColumnName()));
+ }
return schemaChanges;
} else if (change instanceof ModifyPhysicalColumnType) {
- ModifyPhysicalColumnType modify = (ModifyPhysicalColumnType) change;
- LogicalType newColumnType = modify.getNewType().getLogicalType();
- LogicalType oldColumnType = modify.getOldColumn().getDataType().getLogicalType();
- if (newColumnType.isNullable() != oldColumnType.isNullable()) {
+ if (!oldTableNonPhysicalColumnIndex.containsKey(
+ ((ModifyPhysicalColumnType) change).getOldColumn().getName())) {
+ ModifyPhysicalColumnType modify = (ModifyPhysicalColumnType) change;
+ LogicalType newColumnType = modify.getNewType().getLogicalType();
+ LogicalType oldColumnType = modify.getOldColumn().getDataType().getLogicalType();
+ if (newColumnType.isNullable() != oldColumnType.isNullable()) {
+ schemaChanges.add(
+ SchemaChange.updateColumnNullability(
+ modify.getNewColumn().getName(), newColumnType.isNullable()));
+ }
schemaChanges.add(
- SchemaChange.updateColumnNullability(
- modify.getNewColumn().getName(), newColumnType.isNullable()));
+ SchemaChange.updateColumnType(
+ modify.getOldColumn().getName(),
+ LogicalTypeConversion.toDataType(newColumnType)));
}
- schemaChanges.add(
- SchemaChange.updateColumnType(
- modify.getOldColumn().getName(),
- LogicalTypeConversion.toDataType(newColumnType)));
return schemaChanges;
} else if (change instanceof ModifyColumnPosition) {
- ModifyColumnPosition modify = (ModifyColumnPosition) change;
- SchemaChange.Move move =
- getMove(modify.getNewPosition(), modify.getNewColumn().getName());
- schemaChanges.add(SchemaChange.updateColumnPosition(move));
+ if (!oldTableNonPhysicalColumnIndex.containsKey(
+ ((ModifyColumnPosition) change).getOldColumn().getName())) {
+ ModifyColumnPosition modify = (ModifyColumnPosition) change;
+ SchemaChange.Move move =
+ getMove(modify.getNewPosition(), modify.getNewColumn().getName());
+ schemaChanges.add(SchemaChange.updateColumnPosition(move));
+ }
return schemaChanges;
} else if (change instanceof TableChange.ModifyColumnComment) {
- ModifyColumnComment modify = (ModifyColumnComment) change;
- schemaChanges.add(
- SchemaChange.updateColumnComment(
- modify.getNewColumn().getName(), modify.getNewComment()));
+ if (!oldTableNonPhysicalColumnIndex.containsKey(
+ ((ModifyColumnComment) change).getOldColumn().getName())) {
+ ModifyColumnComment modify = (ModifyColumnComment) change;
+ schemaChanges.add(
+ SchemaChange.updateColumnComment(
+ modify.getNewColumn().getName(), modify.getNewComment()));
+ }
return schemaChanges;
} else if (change instanceof ModifyWatermark) {
ModifyWatermark modify = (ModifyWatermark) change;
@@ -499,15 +519,49 @@ public class FlinkCatalog extends AbstractCatalog {
return;
}
- CatalogTable table = getTable(tablePath);
+ Table table;
+ try {
+ table = catalog.getTable(toIdentifier(tablePath));
+ } catch (Catalog.TableNotExistException e) {
+ throw new TableNotExistException(getName(), tablePath);
+ }
- validateAlterTable(table, (CatalogTable) newTable);
+ Preconditions.checkArgument(table instanceof FileStoreTable, "Can't alter system table.");
+ validateAlterTable(toCatalogTable(table), (CatalogTable) newTable);
+ Map<String, Integer> oldTableNonPhysicalColumnIndex =
+ FlinkCatalogPropertiesUtil.nonPhysicalColumns(
+ table.options(), table.rowType().getFieldNames());
List<SchemaChange> changes = new ArrayList<>();
+
+ Map<String, String> schemaOptions =
+ FlinkCatalogPropertiesUtil.serializeNonPhysicalNewColumns(
+ ((ResolvedCatalogBaseTable<?>) newTable).getResolvedSchema());
+ table.options()
+ .forEach(
+ (k, v) -> {
+ if (FlinkCatalogPropertiesUtil.isNonPhysicalColumnKey(k)) {
+ // drop non-physical column
+ if (!schemaOptions.containsKey(k)) {
+ changes.add(SchemaChange.removeOption(k));
+ }
+ }
+ });
+ schemaOptions.forEach(
+ (k, v) -> {
+ // add/modify non-physical column
+ if (!table.options().containsKey(k) || !table.options().get(k).equals(v)) {
+ changes.add(SchemaChange.setOption(k, v));
+ }
+ });
if (null != tableChanges) {
List<SchemaChange> schemaChanges =
tableChanges.stream()
- .flatMap(tableChange -> toSchemaChange(tableChange).stream())
+ .flatMap(
+ tableChange ->
+ toSchemaChange(
+ tableChange, oldTableNonPhysicalColumnIndex)
+ .stream())
.collect(Collectors.toList());
changes.addAll(schemaChanges);
}
@@ -517,7 +571,7 @@ public class FlinkCatalog extends AbstractCatalog {
} catch (Catalog.TableNotExistException
| Catalog.ColumnAlreadyExistException
| Catalog.ColumnNotExistException e) {
- throw new TableNotExistException(getName(), tablePath);
+ throw new CatalogException(e);
}
}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/FlinkCatalogPropertiesUtil.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/FlinkCatalogPropertiesUtil.java
index 011a1a4e0..fb019c6e4 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/FlinkCatalogPropertiesUtil.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/utils/FlinkCatalogPropertiesUtil.java
@@ -18,8 +18,12 @@
package org.apache.paimon.flink.utils;
+import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableSet;
+
import org.apache.flink.table.api.TableColumn;
import org.apache.flink.table.api.WatermarkSpec;
+import org.apache.flink.table.catalog.Column;
+import org.apache.flink.table.catalog.ResolvedSchema;
import org.apache.flink.table.types.DataType;
import org.apache.flink.table.types.logical.utils.LogicalTypeParser;
import org.apache.flink.table.types.utils.TypeConversions;
@@ -27,6 +31,7 @@ import org.apache.flink.table.types.utils.TypeConversions;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -73,6 +78,38 @@ public class FlinkCatalogPropertiesUtil {
return serialized;
}
+ /** Serialize non-physical columns of new api. */
+ public static Map<String, String> serializeNonPhysicalNewColumns(ResolvedSchema schema) {
+ List<Column> nonPhysicalColumns =
+ schema.getColumns().stream()
+ .filter(k -> !k.isPhysical())
+ .collect(Collectors.toList());
+ Map<String, String> serialized = new HashMap<>();
+ List<String> columnNames = schema.getColumnNames();
+ for (Column c : nonPhysicalColumns) {
+ int index = columnNames.indexOf(c.getName());
+ serialized.put(compoundKey(SCHEMA, index, NAME), c.getName());
+ serialized.put(
+ compoundKey(SCHEMA, index, DATA_TYPE),
+ c.getDataType().getLogicalType().asSerializableString());
+ if (c instanceof Column.ComputedColumn) {
+ Column.ComputedColumn computedColumn = (Column.ComputedColumn) c;
+ serialized.put(
+ compoundKey(SCHEMA, index, EXPR),
+ computedColumn.getExpression().asSerializableString());
+ } else {
+ Column.MetadataColumn metadataColumn = (Column.MetadataColumn) c;
+ serialized.put(
+ compoundKey(SCHEMA, index, METADATA),
+ metadataColumn.getMetadataKey().orElse(metadataColumn.getName()));
+ serialized.put(
+ compoundKey(SCHEMA, index, VIRTUAL),
+ Boolean.toString(metadataColumn.isVirtual()));
+ }
+ }
+ return serialized;
+ }
+
public static Map<String, String> serializeWatermarkSpec(WatermarkSpec watermarkSpec) {
Map<String, String> serializedWatermarkSpec = new HashMap<>();
String watermarkPrefix = compoundKey(SCHEMA, WATERMARK, 0);
@@ -89,7 +126,21 @@ public class FlinkCatalogPropertiesUtil {
return serializedWatermarkSpec;
}
- private static final Pattern SCHEMA_COLUMN_NAME_SUFFIX = Pattern.compile("\\d+\\.name");
+ private static final Pattern SCHEMA_COLUMN_NAME_SUFFIX = Pattern.compile("\\d+\\." + NAME);
+ private static final Pattern SCHEMA_COLUMN_METADATA_SUFFIX =
+ Pattern.compile("\\d+\\." + METADATA);
+ private static final Pattern SCHEMA_COLUMN_EXPR_SUFFIX = Pattern.compile("\\d+\\." + EXPR);
+ private static final Pattern SCHEMA_COLUMN_DATATYPE_SUFFIX =
+ Pattern.compile("\\d+\\." + DATA_TYPE);
+ private static final Pattern SCHEMA_COLUMN_VIRTUAL_SUFFIX =
+ Pattern.compile("\\d+\\." + VIRTUAL);
+ private static final Set<Pattern> NON_PHYSICAL_KEY_PATTERNS =
+ ImmutableSet.of(
+ SCHEMA_COLUMN_NAME_SUFFIX,
+ SCHEMA_COLUMN_METADATA_SUFFIX,
+ SCHEMA_COLUMN_EXPR_SUFFIX,
+ SCHEMA_COLUMN_DATATYPE_SUFFIX,
+ SCHEMA_COLUMN_VIRTUAL_SUFFIX);
public static int nonPhysicalColumnsCount(
Map<String, String> tableOptions, List<String> physicalColumns) {
@@ -103,6 +154,36 @@ public class FlinkCatalogPropertiesUtil {
return count;
}
+ public static boolean isNonPhysicalColumnKey(String key) {
+ if (!key.startsWith(SCHEMA)) {
+ return false;
+ }
+ String suffix = key.substring(SCHEMA.length() + 1);
+ for (Pattern pattern : NON_PHYSICAL_KEY_PATTERNS) {
+ if (pattern.matcher(suffix).matches()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public static Map<String, Integer> nonPhysicalColumns(
+ Map<String, String> tableOptions, List<String> physicalColumns) {
+ Map<String, Integer> nonPhysicalColumnIndex = new HashMap<>();
+ for (Map.Entry<String, String> entry : tableOptions.entrySet()) {
+ if (isColumnNameKey(entry.getKey()) && !physicalColumns.contains(entry.getValue())) {
+ String key = entry.getKey();
+ int index =
+ Integer.parseInt(
+ key.substring(
+ SCHEMA.length() + 1,
+ key.indexOf(".", SCHEMA.length() + 1)));
+ nonPhysicalColumnIndex.put(entry.getValue(), index);
+ }
+ }
+ return nonPhysicalColumnIndex;
+ }
+
private static boolean isColumnNameKey(String key) {
return key.startsWith(SCHEMA)
&& SCHEMA_COLUMN_NAME_SUFFIX.matcher(key.substring(SCHEMA.length() + 1)).matches();
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java
index d5590aa6b..920c6a8b3 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/CatalogTableITCase.java
@@ -708,4 +708,24 @@ public class CatalogTableITCase extends CatalogITCaseBase {
"Cannot set streaming-read-overwrite to true when changelog producer "
+ "is full-compaction or lookup because it will read duplicated changes."));
}
+
+ @Test
+ public void testAlterTableNonPhysicalColumn() {
+ sql(
+ "CREATE TABLE T (a INT, c ROW < a INT, d INT> METADATA, b INT, ts TIMESTAMP(3), WATERMARK FOR ts AS ts)");
+ sql("ALTER TABLE T ADD e VARCHAR METADATA");
+ sql("ALTER TABLE T DROP c ");
+ sql("ALTER TABLE T RENAME e TO ee");
+ List<Row> result = sql("SHOW CREATE TABLE T");
+ assertThat(result.get(0).toString())
+ .contains(
+ "CREATE TABLE `PAIMON`.`default`.`T` (\n"
+ + " `a` INT,\n"
+ + " `b` INT,\n"
+ + " `ts` TIMESTAMP(3),\n"
+ + " `ee` VARCHAR(2147483647) METADATA,\n"
+ + " WATERMARK FOR `ts` AS `ts`\n"
+ + ") ")
+ .doesNotContain("schema");
+ }
}