You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by lz...@apache.org on 2022/05/20 09:30:08 UTC

[flink-table-store] branch master updated: [FLINK-27709] Add comment to schema

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git


The following commit(s) were added to refs/heads/master by this push:
     new f93a289  [FLINK-27709] Add comment to schema
f93a289 is described below

commit f93a2897761683f6eb16eb1bb864402aa8b22b57
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Fri May 20 17:30:03 2022 +0800

    [FLINK-27709] Add comment to schema
    
    This closes #130
---
 .../table/store/connector/TableStoreFactory.java   | 21 ++-----------
 .../table/store/connector/FileStoreITCase.java     |  3 +-
 .../flink/table/store/file/schema/Schema.java      | 29 ++++++++++++++++--
 .../table/store/file/schema/SchemaManager.java     | 15 ++++++++--
 .../table/store/file/schema/SchemaSerializer.java  | 13 ++++++++-
 .../table/store/file/schema/UpdateSchema.java      | 34 ++++++++++++++++++++--
 .../table/store/file/schema/SchemaManagerTest.java | 13 ++++++---
 .../store/file/schema/SchemaSerializationTest.java |  3 +-
 .../flink/table/store/file/schema/SchemaTest.java  |  2 +-
 9 files changed, 100 insertions(+), 33 deletions(-)

diff --git a/flink-table-store-connector/src/main/java/org/apache/flink/table/store/connector/TableStoreFactory.java b/flink-table-store-connector/src/main/java/org/apache/flink/table/store/connector/TableStoreFactory.java
index 38df263..a61e33e 100644
--- a/flink-table-store-connector/src/main/java/org/apache/flink/table/store/connector/TableStoreFactory.java
+++ b/flink-table-store-connector/src/main/java/org/apache/flink/table/store/connector/TableStoreFactory.java
@@ -28,8 +28,6 @@ import org.apache.flink.core.fs.Path;
 import org.apache.flink.table.api.TableException;
 import org.apache.flink.table.api.ValidationException;
 import org.apache.flink.table.catalog.CatalogPartitionSpec;
-import org.apache.flink.table.catalog.ResolvedCatalogTable;
-import org.apache.flink.table.catalog.ResolvedSchema;
 import org.apache.flink.table.factories.DynamicTableSinkFactory;
 import org.apache.flink.table.factories.DynamicTableSourceFactory;
 import org.apache.flink.table.factories.FactoryUtil;
@@ -52,7 +50,6 @@ import org.apache.flink.util.Preconditions;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -133,7 +130,8 @@ public class TableStoreFactory
                         .path(context.getObjectIdentifier());
         // TODO pass lock
         try {
-            new SchemaManager(tablePath).commitNewVersion(newUpdateSchema(context));
+            new SchemaManager(tablePath)
+                    .commitNewVersion(UpdateSchema.fromCatalogTable(context.getCatalogTable()));
         } catch (IllegalStateException e) {
             throw e;
         } catch (Exception e) {
@@ -271,19 +269,6 @@ public class TableStoreFactory
                                 Map.Entry::getValue));
     }
 
-    private UpdateSchema newUpdateSchema(Context context) {
-        ResolvedCatalogTable catalogTable = context.getCatalogTable();
-        ResolvedSchema schema = catalogTable.getResolvedSchema();
-        RowType rowType = (RowType) schema.toPhysicalRowDataType().getLogicalType();
-        List<String> primaryKeys = new ArrayList<>();
-        if (schema.getPrimaryKey().isPresent()) {
-            primaryKeys = schema.getPrimaryKey().get().getColumns();
-        }
-
-        return new UpdateSchema(
-                rowType, catalogTable.getPartitionKeys(), primaryKeys, catalogTable.getOptions());
-    }
-
     @VisibleForTesting
     TableStore buildTableStore(Context context) {
         TableStore store =
@@ -293,7 +278,7 @@ public class TableStoreFactory
 
         Schema schema = store.schema();
 
-        UpdateSchema updateSchema = newUpdateSchema(context);
+        UpdateSchema updateSchema = UpdateSchema.fromCatalogTable(context.getCatalogTable());
 
         RowType rowType = updateSchema.rowType();
         List<String> partitionKeys = updateSchema.partitionKeys();
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/FileStoreITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/FileStoreITCase.java
index 601c8ba..e41f3dc 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/FileStoreITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/FileStoreITCase.java
@@ -361,7 +361,8 @@ public class FileStoreITCase extends AbstractTestBase {
                                 Arrays.stream(primaryKey)
                                         .mapToObj(i -> TABLE_TYPE.getFieldNames().get(i))
                                         .collect(Collectors.toList()),
-                                options.toMap()));
+                                options.toMap(),
+                                ""));
         return retryArtificialException(() -> new TableStore(identifier, options));
     }
 
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/Schema.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/Schema.java
index 134af4c..0dbc08c 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/Schema.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/Schema.java
@@ -18,12 +18,14 @@
 
 package org.apache.flink.table.store.file.schema;
 
+import org.apache.flink.table.api.TableSchema;
 import org.apache.flink.table.store.file.utils.JsonSerdeUtil;
 import org.apache.flink.table.types.logical.ArrayType;
 import org.apache.flink.table.types.logical.LogicalType;
 import org.apache.flink.table.types.logical.MapType;
 import org.apache.flink.table.types.logical.MultisetType;
 import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.types.utils.TypeConversions;
 import org.apache.flink.util.Preconditions;
 
 import java.util.ArrayList;
@@ -52,19 +54,23 @@ public class Schema {
 
     private final Map<String, String> options;
 
+    private final String comment;
+
     public Schema(
             long id,
             List<DataField> fields,
             int highestFieldId,
             List<String> partitionKeys,
             List<String> primaryKeys,
-            Map<String, String> options) {
+            Map<String, String> options,
+            String comment) {
         this.id = id;
         this.fields = fields;
         this.highestFieldId = highestFieldId;
         this.partitionKeys = partitionKeys;
         this.primaryKeys = primaryKeys;
         this.options = Collections.unmodifiableMap(options);
+        this.comment = comment;
 
         // try to trim to validate primary keys
         trimmedPrimaryKeys();
@@ -118,6 +124,10 @@ public class Schema {
         return options;
     }
 
+    public String comment() {
+        return comment;
+    }
+
     public RowType logicalRowType() {
         return (RowType) new RowDataType(fields).logicalType;
     }
@@ -139,12 +149,25 @@ public class Schema {
         return Objects.equals(fields, schema.fields)
                 && Objects.equals(partitionKeys, schema.partitionKeys)
                 && Objects.equals(primaryKeys, schema.primaryKeys)
-                && Objects.equals(options, schema.options);
+                && Objects.equals(options, schema.options)
+                && Objects.equals(comment, schema.comment);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(fields, partitionKeys, primaryKeys, options);
+        return Objects.hash(fields, partitionKeys, primaryKeys, options, comment);
+    }
+
+    public TableSchema getTableSchema() {
+        TableSchema.Builder builder = TableSchema.builder();
+        for (DataField field : fields) {
+            builder.field(
+                    field.name(), TypeConversions.fromLogicalToDataType(field.type().logicalType));
+        }
+        if (primaryKeys.size() > 0) {
+            builder.primaryKey(primaryKeys.toArray(new String[0]));
+        }
+        return builder.build();
     }
 
     public static List<DataField> newFields(RowType rowType) {
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/SchemaManager.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/SchemaManager.java
index 50610d2..ac835fc 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/SchemaManager.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/SchemaManager.java
@@ -68,9 +68,13 @@ public class SchemaManager {
 
     /** List all schema. */
     public List<Schema> listAll() {
+        return listAllIds().stream().map(this::schema).collect(Collectors.toList());
+    }
+
+    /** List all schema IDs. */
+    public List<Long> listAllIds() {
         try {
             return listVersionedFiles(schemaDirectory(), SCHEMA_PREFIX)
-                    .map(this::schema)
                     .collect(Collectors.toList());
         } catch (IOException e) {
             throw new UncheckedIOException(e);
@@ -117,7 +121,14 @@ public class SchemaManager {
             }
 
             Schema schema =
-                    new Schema(id, fields, highestFieldId, partitionKeys, primaryKeys, options);
+                    new Schema(
+                            id,
+                            fields,
+                            highestFieldId,
+                            partitionKeys,
+                            primaryKeys,
+                            options,
+                            updateSchema.comment());
 
             Path temp = toTmpSchemaPath(id);
             Path finalFile = toSchemaPath(id);
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/SchemaSerializer.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/SchemaSerializer.java
index d200149..089c804 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/SchemaSerializer.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/SchemaSerializer.java
@@ -19,6 +19,7 @@
 package org.apache.flink.table.store.file.schema;
 
 import org.apache.flink.table.store.file.utils.JsonSerializer;
+import org.apache.flink.util.StringUtils;
 
 import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator;
 import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode;
@@ -67,6 +68,10 @@ public class SchemaSerializer implements JsonSerializer<Schema> {
         }
         generator.writeEndObject();
 
+        if (!StringUtils.isNullOrWhitespaceOnly(schema.comment())) {
+            generator.writeStringField("comment", schema.comment());
+        }
+
         generator.writeEndObject();
     }
 
@@ -102,6 +107,12 @@ public class SchemaSerializer implements JsonSerializer<Schema> {
             options.put(key, optionsJson.get(key).asText());
         }
 
-        return new Schema(id, fields, highestFieldId, partitionKeys, primaryKeys, options);
+        JsonNode commentNode = node.get("comment");
+        String comment = "";
+        if (commentNode != null) {
+            comment = commentNode.asText();
+        }
+
+        return new Schema(id, fields, highestFieldId, partitionKeys, primaryKeys, options, comment);
     }
 }
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/UpdateSchema.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/UpdateSchema.java
index ef0ff7c..ed7ea72 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/UpdateSchema.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/UpdateSchema.java
@@ -18,8 +18,12 @@
 
 package org.apache.flink.table.store.file.schema;
 
+import org.apache.flink.table.catalog.ResolvedCatalogTable;
+import org.apache.flink.table.catalog.ResolvedSchema;
 import org.apache.flink.table.types.logical.RowType;
 
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -34,15 +38,35 @@ public class UpdateSchema {
 
     private final Map<String, String> options;
 
+    private final String comment;
+
+    public static UpdateSchema fromCatalogTable(ResolvedCatalogTable catalogTable) {
+        ResolvedSchema schema = catalogTable.getResolvedSchema();
+        RowType rowType = (RowType) schema.toPhysicalRowDataType().getLogicalType();
+        List<String> primaryKeys = new ArrayList<>();
+        if (schema.getPrimaryKey().isPresent()) {
+            primaryKeys = schema.getPrimaryKey().get().getColumns();
+        }
+
+        return new UpdateSchema(
+                rowType,
+                catalogTable.getPartitionKeys(),
+                primaryKeys,
+                catalogTable.getOptions(),
+                catalogTable.getComment());
+    }
+
     public UpdateSchema(
             RowType rowType,
             List<String> partitionKeys,
             List<String> primaryKeys,
-            Map<String, String> options) {
+            Map<String, String> options,
+            String comment) {
         this.rowType = rowType;
         this.partitionKeys = partitionKeys;
         this.primaryKeys = primaryKeys;
-        this.options = options;
+        this.options = new HashMap<>(options);
+        this.comment = comment;
     }
 
     public RowType rowType() {
@@ -61,6 +85,10 @@ public class UpdateSchema {
         return options;
     }
 
+    public String comment() {
+        return comment;
+    }
+
     @Override
     public String toString() {
         return "UpdateSchema{"
@@ -72,6 +100,8 @@ public class UpdateSchema {
                 + primaryKeys
                 + ", options="
                 + options
+                + ", comment="
+                + comment
                 + '}';
     }
 }
diff --git a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaManagerTest.java b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaManagerTest.java
index f4984a7..90084c0 100644
--- a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaManagerTest.java
+++ b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaManagerTest.java
@@ -59,7 +59,7 @@ public class SchemaManagerTest {
     private final Map<String, String> options = Collections.singletonMap("key", "value");
     private final RowType rowType = RowType.of(new IntType(), new BigIntType(), new VarCharType());
     private final UpdateSchema updateSchema =
-            new UpdateSchema(rowType, partitionKeys, primaryKeys, options);
+            new UpdateSchema(rowType, partitionKeys, primaryKeys, options, "");
 
     @BeforeEach
     public void beforeEach() throws IOException {
@@ -106,7 +106,7 @@ public class SchemaManagerTest {
         retryArtificialException(() -> manager.commitNewVersion(updateSchema));
         Map<String, String> newOptions = Collections.singletonMap("new_k", "new_v");
         UpdateSchema updateSchema =
-                new UpdateSchema(rowType, partitionKeys, primaryKeys, newOptions);
+                new UpdateSchema(rowType, partitionKeys, primaryKeys, newOptions, "my_comment_2");
         retryArtificialException(() -> manager.commitNewVersion(updateSchema));
         Optional<Schema> latest = retryArtificialException(() -> manager.latest());
         assertThat(latest.isPresent()).isTrue();
@@ -117,7 +117,8 @@ public class SchemaManagerTest {
     public void testUpdateSchema() throws Exception {
         retryArtificialException(() -> manager.commitNewVersion(updateSchema));
         RowType newType = RowType.of(new IntType(), new BigIntType());
-        UpdateSchema updateSchema = new UpdateSchema(newType, partitionKeys, primaryKeys, options);
+        UpdateSchema updateSchema =
+                new UpdateSchema(newType, partitionKeys, primaryKeys, options, "my_comment_3");
         assertThrows(
                 UnsupportedOperationException.class,
                 () -> retryArtificialException(() -> manager.commitNewVersion(updateSchema)));
@@ -143,7 +144,11 @@ public class SchemaManagerTest {
                                 options.put("id", String.valueOf(id));
                                 UpdateSchema updateSchema =
                                         new UpdateSchema(
-                                                rowType, partitionKeys, primaryKeys, options);
+                                                rowType,
+                                                partitionKeys,
+                                                primaryKeys,
+                                                options,
+                                                "my_comment_4");
                                 try {
                                     retryArtificialException(
                                             () -> manager.commitNewVersion(updateSchema));
diff --git a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaSerializationTest.java b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaSerializationTest.java
index bd214ab..4aae8ee 100644
--- a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaSerializationTest.java
+++ b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaSerializationTest.java
@@ -50,7 +50,8 @@ public class SchemaSerializationTest {
         options.put("option-1", "value-1");
         options.put("option-2", "value-2");
 
-        Schema schema = new Schema(1, fields, 10, partitionKeys, primaryKeys, options);
+        Schema schema =
+                new Schema(1, fields, 10, partitionKeys, primaryKeys, options, "my_comment");
         String serialized = JsonSerdeUtil.toJson(schema);
 
         Schema deserialized = JsonSerdeUtil.fromJson(serialized, Schema.class);
diff --git a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaTest.java b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaTest.java
index 85d4078..3d3bea7 100644
--- a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaTest.java
+++ b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaTest.java
@@ -52,7 +52,7 @@ public class SchemaTest {
 
         Assertions.assertThrows(
                 IllegalStateException.class,
-                () -> new Schema(1, fields, 10, partitionKeys, primaryKeys, options));
+                () -> new Schema(1, fields, 10, partitionKeys, primaryKeys, options, ""));
     }
 
     @Test