You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by lz...@apache.org on 2022/05/20 09:30:08 UTC
[flink-table-store] branch master updated: [FLINK-27709] Add comment to schema
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git
The following commit(s) were added to refs/heads/master by this push:
new f93a289 [FLINK-27709] Add comment to schema
f93a289 is described below
commit f93a2897761683f6eb16eb1bb864402aa8b22b57
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Fri May 20 17:30:03 2022 +0800
[FLINK-27709] Add comment to schema
This closes #130
---
.../table/store/connector/TableStoreFactory.java | 21 ++-----------
.../table/store/connector/FileStoreITCase.java | 3 +-
.../flink/table/store/file/schema/Schema.java | 29 ++++++++++++++++--
.../table/store/file/schema/SchemaManager.java | 15 ++++++++--
.../table/store/file/schema/SchemaSerializer.java | 13 ++++++++-
.../table/store/file/schema/UpdateSchema.java | 34 ++++++++++++++++++++--
.../table/store/file/schema/SchemaManagerTest.java | 13 ++++++---
.../store/file/schema/SchemaSerializationTest.java | 3 +-
.../flink/table/store/file/schema/SchemaTest.java | 2 +-
9 files changed, 100 insertions(+), 33 deletions(-)
diff --git a/flink-table-store-connector/src/main/java/org/apache/flink/table/store/connector/TableStoreFactory.java b/flink-table-store-connector/src/main/java/org/apache/flink/table/store/connector/TableStoreFactory.java
index 38df263..a61e33e 100644
--- a/flink-table-store-connector/src/main/java/org/apache/flink/table/store/connector/TableStoreFactory.java
+++ b/flink-table-store-connector/src/main/java/org/apache/flink/table/store/connector/TableStoreFactory.java
@@ -28,8 +28,6 @@ import org.apache.flink.core.fs.Path;
import org.apache.flink.table.api.TableException;
import org.apache.flink.table.api.ValidationException;
import org.apache.flink.table.catalog.CatalogPartitionSpec;
-import org.apache.flink.table.catalog.ResolvedCatalogTable;
-import org.apache.flink.table.catalog.ResolvedSchema;
import org.apache.flink.table.factories.DynamicTableSinkFactory;
import org.apache.flink.table.factories.DynamicTableSourceFactory;
import org.apache.flink.table.factories.FactoryUtil;
@@ -52,7 +50,6 @@ import org.apache.flink.util.Preconditions;
import java.io.IOException;
import java.io.UncheckedIOException;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -133,7 +130,8 @@ public class TableStoreFactory
.path(context.getObjectIdentifier());
// TODO pass lock
try {
- new SchemaManager(tablePath).commitNewVersion(newUpdateSchema(context));
+ new SchemaManager(tablePath)
+ .commitNewVersion(UpdateSchema.fromCatalogTable(context.getCatalogTable()));
} catch (IllegalStateException e) {
throw e;
} catch (Exception e) {
@@ -271,19 +269,6 @@ public class TableStoreFactory
Map.Entry::getValue));
}
- private UpdateSchema newUpdateSchema(Context context) {
- ResolvedCatalogTable catalogTable = context.getCatalogTable();
- ResolvedSchema schema = catalogTable.getResolvedSchema();
- RowType rowType = (RowType) schema.toPhysicalRowDataType().getLogicalType();
- List<String> primaryKeys = new ArrayList<>();
- if (schema.getPrimaryKey().isPresent()) {
- primaryKeys = schema.getPrimaryKey().get().getColumns();
- }
-
- return new UpdateSchema(
- rowType, catalogTable.getPartitionKeys(), primaryKeys, catalogTable.getOptions());
- }
-
@VisibleForTesting
TableStore buildTableStore(Context context) {
TableStore store =
@@ -293,7 +278,7 @@ public class TableStoreFactory
Schema schema = store.schema();
- UpdateSchema updateSchema = newUpdateSchema(context);
+ UpdateSchema updateSchema = UpdateSchema.fromCatalogTable(context.getCatalogTable());
RowType rowType = updateSchema.rowType();
List<String> partitionKeys = updateSchema.partitionKeys();
diff --git a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/FileStoreITCase.java b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/FileStoreITCase.java
index 601c8ba..e41f3dc 100644
--- a/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/FileStoreITCase.java
+++ b/flink-table-store-connector/src/test/java/org/apache/flink/table/store/connector/FileStoreITCase.java
@@ -361,7 +361,8 @@ public class FileStoreITCase extends AbstractTestBase {
Arrays.stream(primaryKey)
.mapToObj(i -> TABLE_TYPE.getFieldNames().get(i))
.collect(Collectors.toList()),
- options.toMap()));
+ options.toMap(),
+ ""));
return retryArtificialException(() -> new TableStore(identifier, options));
}
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/Schema.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/Schema.java
index 134af4c..0dbc08c 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/Schema.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/Schema.java
@@ -18,12 +18,14 @@
package org.apache.flink.table.store.file.schema;
+import org.apache.flink.table.api.TableSchema;
import org.apache.flink.table.store.file.utils.JsonSerdeUtil;
import org.apache.flink.table.types.logical.ArrayType;
import org.apache.flink.table.types.logical.LogicalType;
import org.apache.flink.table.types.logical.MapType;
import org.apache.flink.table.types.logical.MultisetType;
import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.table.types.utils.TypeConversions;
import org.apache.flink.util.Preconditions;
import java.util.ArrayList;
@@ -52,19 +54,23 @@ public class Schema {
private final Map<String, String> options;
+ private final String comment;
+
public Schema(
long id,
List<DataField> fields,
int highestFieldId,
List<String> partitionKeys,
List<String> primaryKeys,
- Map<String, String> options) {
+ Map<String, String> options,
+ String comment) {
this.id = id;
this.fields = fields;
this.highestFieldId = highestFieldId;
this.partitionKeys = partitionKeys;
this.primaryKeys = primaryKeys;
this.options = Collections.unmodifiableMap(options);
+ this.comment = comment;
// try to trim to validate primary keys
trimmedPrimaryKeys();
@@ -118,6 +124,10 @@ public class Schema {
return options;
}
+ public String comment() {
+ return comment;
+ }
+
public RowType logicalRowType() {
return (RowType) new RowDataType(fields).logicalType;
}
@@ -139,12 +149,25 @@ public class Schema {
return Objects.equals(fields, schema.fields)
&& Objects.equals(partitionKeys, schema.partitionKeys)
&& Objects.equals(primaryKeys, schema.primaryKeys)
- && Objects.equals(options, schema.options);
+ && Objects.equals(options, schema.options)
+ && Objects.equals(comment, schema.comment);
}
@Override
public int hashCode() {
- return Objects.hash(fields, partitionKeys, primaryKeys, options);
+ return Objects.hash(fields, partitionKeys, primaryKeys, options, comment);
+ }
+
+ public TableSchema getTableSchema() {
+ TableSchema.Builder builder = TableSchema.builder();
+ for (DataField field : fields) {
+ builder.field(
+ field.name(), TypeConversions.fromLogicalToDataType(field.type().logicalType));
+ }
+ if (primaryKeys.size() > 0) {
+ builder.primaryKey(primaryKeys.toArray(new String[0]));
+ }
+ return builder.build();
}
public static List<DataField> newFields(RowType rowType) {
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/SchemaManager.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/SchemaManager.java
index 50610d2..ac835fc 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/SchemaManager.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/SchemaManager.java
@@ -68,9 +68,13 @@ public class SchemaManager {
/** List all schema. */
public List<Schema> listAll() {
+ return listAllIds().stream().map(this::schema).collect(Collectors.toList());
+ }
+
+ /** List all schema IDs. */
+ public List<Long> listAllIds() {
try {
return listVersionedFiles(schemaDirectory(), SCHEMA_PREFIX)
- .map(this::schema)
.collect(Collectors.toList());
} catch (IOException e) {
throw new UncheckedIOException(e);
@@ -117,7 +121,14 @@ public class SchemaManager {
}
Schema schema =
- new Schema(id, fields, highestFieldId, partitionKeys, primaryKeys, options);
+ new Schema(
+ id,
+ fields,
+ highestFieldId,
+ partitionKeys,
+ primaryKeys,
+ options,
+ updateSchema.comment());
Path temp = toTmpSchemaPath(id);
Path finalFile = toSchemaPath(id);
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/SchemaSerializer.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/SchemaSerializer.java
index d200149..089c804 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/SchemaSerializer.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/SchemaSerializer.java
@@ -19,6 +19,7 @@
package org.apache.flink.table.store.file.schema;
import org.apache.flink.table.store.file.utils.JsonSerializer;
+import org.apache.flink.util.StringUtils;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode;
@@ -67,6 +68,10 @@ public class SchemaSerializer implements JsonSerializer<Schema> {
}
generator.writeEndObject();
+ if (!StringUtils.isNullOrWhitespaceOnly(schema.comment())) {
+ generator.writeStringField("comment", schema.comment());
+ }
+
generator.writeEndObject();
}
@@ -102,6 +107,12 @@ public class SchemaSerializer implements JsonSerializer<Schema> {
options.put(key, optionsJson.get(key).asText());
}
- return new Schema(id, fields, highestFieldId, partitionKeys, primaryKeys, options);
+ JsonNode commentNode = node.get("comment");
+ String comment = "";
+ if (commentNode != null) {
+ comment = commentNode.asText();
+ }
+
+ return new Schema(id, fields, highestFieldId, partitionKeys, primaryKeys, options, comment);
}
}
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/UpdateSchema.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/UpdateSchema.java
index ef0ff7c..ed7ea72 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/UpdateSchema.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/schema/UpdateSchema.java
@@ -18,8 +18,12 @@
package org.apache.flink.table.store.file.schema;
+import org.apache.flink.table.catalog.ResolvedCatalogTable;
+import org.apache.flink.table.catalog.ResolvedSchema;
import org.apache.flink.table.types.logical.RowType;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -34,15 +38,35 @@ public class UpdateSchema {
private final Map<String, String> options;
+ private final String comment;
+
+ public static UpdateSchema fromCatalogTable(ResolvedCatalogTable catalogTable) {
+ ResolvedSchema schema = catalogTable.getResolvedSchema();
+ RowType rowType = (RowType) schema.toPhysicalRowDataType().getLogicalType();
+ List<String> primaryKeys = new ArrayList<>();
+ if (schema.getPrimaryKey().isPresent()) {
+ primaryKeys = schema.getPrimaryKey().get().getColumns();
+ }
+
+ return new UpdateSchema(
+ rowType,
+ catalogTable.getPartitionKeys(),
+ primaryKeys,
+ catalogTable.getOptions(),
+ catalogTable.getComment());
+ }
+
public UpdateSchema(
RowType rowType,
List<String> partitionKeys,
List<String> primaryKeys,
- Map<String, String> options) {
+ Map<String, String> options,
+ String comment) {
this.rowType = rowType;
this.partitionKeys = partitionKeys;
this.primaryKeys = primaryKeys;
- this.options = options;
+ this.options = new HashMap<>(options);
+ this.comment = comment;
}
public RowType rowType() {
@@ -61,6 +85,10 @@ public class UpdateSchema {
return options;
}
+ public String comment() {
+ return comment;
+ }
+
@Override
public String toString() {
return "UpdateSchema{"
@@ -72,6 +100,8 @@ public class UpdateSchema {
+ primaryKeys
+ ", options="
+ options
+ + ", comment="
+ + comment
+ '}';
}
}
diff --git a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaManagerTest.java b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaManagerTest.java
index f4984a7..90084c0 100644
--- a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaManagerTest.java
+++ b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaManagerTest.java
@@ -59,7 +59,7 @@ public class SchemaManagerTest {
private final Map<String, String> options = Collections.singletonMap("key", "value");
private final RowType rowType = RowType.of(new IntType(), new BigIntType(), new VarCharType());
private final UpdateSchema updateSchema =
- new UpdateSchema(rowType, partitionKeys, primaryKeys, options);
+ new UpdateSchema(rowType, partitionKeys, primaryKeys, options, "");
@BeforeEach
public void beforeEach() throws IOException {
@@ -106,7 +106,7 @@ public class SchemaManagerTest {
retryArtificialException(() -> manager.commitNewVersion(updateSchema));
Map<String, String> newOptions = Collections.singletonMap("new_k", "new_v");
UpdateSchema updateSchema =
- new UpdateSchema(rowType, partitionKeys, primaryKeys, newOptions);
+ new UpdateSchema(rowType, partitionKeys, primaryKeys, newOptions, "my_comment_2");
retryArtificialException(() -> manager.commitNewVersion(updateSchema));
Optional<Schema> latest = retryArtificialException(() -> manager.latest());
assertThat(latest.isPresent()).isTrue();
@@ -117,7 +117,8 @@ public class SchemaManagerTest {
public void testUpdateSchema() throws Exception {
retryArtificialException(() -> manager.commitNewVersion(updateSchema));
RowType newType = RowType.of(new IntType(), new BigIntType());
- UpdateSchema updateSchema = new UpdateSchema(newType, partitionKeys, primaryKeys, options);
+ UpdateSchema updateSchema =
+ new UpdateSchema(newType, partitionKeys, primaryKeys, options, "my_comment_3");
assertThrows(
UnsupportedOperationException.class,
() -> retryArtificialException(() -> manager.commitNewVersion(updateSchema)));
@@ -143,7 +144,11 @@ public class SchemaManagerTest {
options.put("id", String.valueOf(id));
UpdateSchema updateSchema =
new UpdateSchema(
- rowType, partitionKeys, primaryKeys, options);
+ rowType,
+ partitionKeys,
+ primaryKeys,
+ options,
+ "my_comment_4");
try {
retryArtificialException(
() -> manager.commitNewVersion(updateSchema));
diff --git a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaSerializationTest.java b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaSerializationTest.java
index bd214ab..4aae8ee 100644
--- a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaSerializationTest.java
+++ b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaSerializationTest.java
@@ -50,7 +50,8 @@ public class SchemaSerializationTest {
options.put("option-1", "value-1");
options.put("option-2", "value-2");
- Schema schema = new Schema(1, fields, 10, partitionKeys, primaryKeys, options);
+ Schema schema =
+ new Schema(1, fields, 10, partitionKeys, primaryKeys, options, "my_comment");
String serialized = JsonSerdeUtil.toJson(schema);
Schema deserialized = JsonSerdeUtil.fromJson(serialized, Schema.class);
diff --git a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaTest.java b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaTest.java
index 85d4078..3d3bea7 100644
--- a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaTest.java
+++ b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/schema/SchemaTest.java
@@ -52,7 +52,7 @@ public class SchemaTest {
Assertions.assertThrows(
IllegalStateException.class,
- () -> new Schema(1, fields, 10, partitionKeys, primaryKeys, options));
+ () -> new Schema(1, fields, 10, partitionKeys, primaryKeys, options, ""));
}
@Test