You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by "hejiay (via GitHub)" <gi...@apache.org> on 2023/05/30 02:52:15 UTC
[GitHub] [inlong] hejiay commented on a diff in pull request #8120: [INLONG-7959][Sort] Dynamic schema evolution support delete and update columns when sink to Iceberg
hejiay commented on code in PR #8120:
URL: https://github.com/apache/inlong/pull/8120#discussion_r1209656737
##########
inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/iceberg/src/main/java/org/apache/inlong/sort/iceberg/sink/multiple/SchemaChangeUtils.java:
##########
@@ -52,49 +54,85 @@ public class SchemaChangeUtils {
static List<TableChange> diffSchema(Schema oldSchema, Schema newSchema) {
List<String> oldFields = oldSchema.columns().stream().map(NestedField::name).collect(Collectors.toList());
List<String> newFields = newSchema.columns().stream().map(NestedField::name).collect(Collectors.toList());
- int oi = 0;
- int ni = 0;
+ Set<String> oldFieldSet = new HashSet<>(oldFields);
+ Set<String> newFieldSet = new HashSet<>(newFields);
+
+ Set<String> intersectColSet = Sets.intersection(oldFieldSet, newFieldSet);
+ Set<String> colsToDelete = Sets.difference(oldFieldSet, newFieldSet);
+ Set<String> colsToAdd = Sets.difference(newFieldSet, oldFieldSet);
+
List<TableChange> tableChanges = new ArrayList<>();
- while (ni < newFields.size()) {
- if (oi < oldFields.size() && oldFields.get(oi).equals(newFields.get(ni))) {
- oi++;
- ni++;
- } else {
- NestedField newField = newSchema.findField(newFields.get(ni));
+
+ // step0: judge whether unknown change
+ // just diff two different schema can not distinguish(add + delete) vs modify
+ // Example first [a, b, c] -> then delete c [a, b] -> add d [a, b, d], currently it is only judged as unknown
+ // change.
+ // In next version,we will judge it is [delete and add] or rename by using information extracted from DDL
+ if (!colsToDelete.isEmpty() && !colsToAdd.isEmpty()) {
+ tableChanges.add(new UnknownColumnChange(
+ String.format(" old schema: [%s] and new schema: [%s], it is unknown column change",
+ oldSchema.toString(), newSchema.toString())));
+ return tableChanges;
+ }
+
+ // step1: judge whether column type change
+ for (String colName : intersectColSet) {
+ NestedField oldField = oldSchema.findField(colName);
+ NestedField newField = newSchema.findField(colName);
+ if (!oldField.type().equals(newField.type()) || !oldField.doc().equals(newField.doc())) {
tableChanges.add(
- new AddColumn(
+ new TableChange.UpdateColumn(
new String[]{newField.name()},
FlinkSchemaUtil.convert(newField.type()),
!newField.isRequired(),
- newField.doc(),
- ni == 0 ? ColumnPosition.first() : ColumnPosition.after(newFields.get(ni - 1))));
- ni++;
+ newField.doc()));
}
}
- if (oi != oldFields.size()) {
- tableChanges.clear();
- tableChanges.add(
- new UnknownColumnChange(
- String.format("Unsupported schema update.\n"
- + "oldSchema:\n%s\n, newSchema:\n %s", oldSchema, newSchema)));
+ // step2: judge whether delete column
+ for (String colName : oldFields) {
+ if (colsToDelete.contains(colName)) {
+ tableChanges.add(
+ new TableChange.DeleteColumn(
+ new String[]{colName}));
+ }
}
+ // step3: judge whether add column
+ if (!colsToAdd.isEmpty()) {
+ for (int i = 0; i < newFields.size(); i++) {
Review Comment:
In order to get the position of the added field
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@inlong.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org