You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ja...@apache.org on 2020/08/12 03:13:22 UTC
[flink] branch master updated: [FLINK-18844][json][maxwell] Support
maxwell-json format to read Maxwell changelogs
This is an automated email from the ASF dual-hosted git repository.
jark pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 2d75031 [FLINK-18844][json][maxwell] Support maxwell-json format to read Maxwell changelogs
2d75031 is described below
commit 2d75031d3d6bdc11c7b3fcf25cd43f396677003c
Author: dijie <nj...@gmail.com>
AuthorDate: Wed Aug 12 11:11:58 2020 +0800
[FLINK-18844][json][maxwell] Support maxwell-json format to read Maxwell changelogs
This closes #13090
---
.../maxwell/MaxwellJsonDeserializationSchema.java | 179 +++++++++++++++++++
.../json/maxwell/MaxwellJsonFormatFactory.java | 139 +++++++++++++++
.../maxwell/MaxwellJsonSerializationSchema.java | 111 ++++++++++++
.../org.apache.flink.table.factories.Factory | 1 +
.../json/maxwell/MaxwellJsonFormatFactoryTest.java | 159 +++++++++++++++++
.../json/maxwell/MaxwellJsonSerDerTest.java | 198 +++++++++++++++++++++
.../flink-json/src/test/resources/maxwell-data.txt | 20 +++
7 files changed, 807 insertions(+)
diff --git a/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/maxwell/MaxwellJsonDeserializationSchema.java b/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/maxwell/MaxwellJsonDeserializationSchema.java
new file mode 100644
index 0000000..c34ab55
--- /dev/null
+++ b/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/maxwell/MaxwellJsonDeserializationSchema.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.formats.json.maxwell;
+
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.formats.json.JsonRowDataDeserializationSchema;
+import org.apache.flink.formats.json.TimestampFormat;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static java.lang.String.format;
+import static org.apache.flink.table.types.utils.TypeConversions.fromLogicalToDataType;
+
+/**
+ * Deserialization schema from Maxwell JSON to Flink Table/SQL internal data structure {@link RowData}.
+ * The deserialization schema knows Maxwell's schema definition and can extract the database data
+ * and convert into {@link RowData} with {@link RowKind}.
+ *
+ * <p>Deserializes a <code>byte[]</code> message as a JSON object and reads
+ * the specified fields.
+ *
+ * <p>Failures during deserialization are forwarded as wrapped IOExceptions.
+ *
+ * @see <a href="http://maxwells-daemon.io/">Maxwell</a>
+ */
+public class MaxwellJsonDeserializationSchema implements DeserializationSchema<RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final String OP_INSERT = "insert";
+ private static final String OP_UPDATE = "update";
+ private static final String OP_DELETE = "delete";
+
+ /** The deserializer to deserialize Maxwell JSON data. */
+ private final JsonRowDataDeserializationSchema jsonDeserializer;
+
+ /** TypeInformation of the produced {@link RowData}. **/
+ private final TypeInformation<RowData> resultTypeInfo;
+
+ /** Flag indicating whether to ignore invalid fields/rows (default: throw an exception). */
+ private final boolean ignoreParseErrors;
+
+ /** Number of fields. */
+ private final int fieldCount;
+
+ public MaxwellJsonDeserializationSchema(RowType rowType,
+ TypeInformation<RowData> resultTypeInfo,
+ boolean ignoreParseErrors,
+ TimestampFormat timestampFormatOption) {
+ this.resultTypeInfo = resultTypeInfo;
+ this.ignoreParseErrors = ignoreParseErrors;
+ this.fieldCount = rowType.getFieldCount();
+ this.jsonDeserializer = new JsonRowDataDeserializationSchema(
+ createJsonRowType(fromLogicalToDataType(rowType)),
+ // the result type is never used, so it's fine to pass in Canal's result type
+ resultTypeInfo,
+ false, // ignoreParseErrors already contains the functionality of failOnMissingField
+ ignoreParseErrors,
+ timestampFormatOption);
+ }
+
+ @Override
+ public RowData deserialize(byte[] message) throws IOException {
+ throw new RuntimeException(
+ "Please invoke DeserializationSchema#deserialize(byte[], Collector<RowData>) instead.");
+ }
+
+ @Override
+ public void deserialize(byte[] message, Collector<RowData> out) throws IOException {
+ try {
+ RowData row = jsonDeserializer.deserialize(message);
+ String type = row.getString(2).toString(); // "type" field
+ if (OP_INSERT.equals(type)) {
+ // "data" field is a row, contains inserted rows
+ RowData insert = row.getRow(0, fieldCount);
+ insert.setRowKind(RowKind.INSERT);
+ out.collect(insert);
+ } else if (OP_UPDATE.equals(type)) {
+ // "data" field is a row, contains new rows
+ // "old" field is an array of row, contains old values
+ // the underlying JSON deserialization schema always produce GenericRowData.
+ GenericRowData after = (GenericRowData) row.getRow(0, fieldCount); // "data" field
+ GenericRowData before = (GenericRowData) row.getRow(1, fieldCount); // "old" field
+ for (int f = 0; f < fieldCount; f++) {
+ if (before.isNullAt(f)) {
+ // not null fields in "old" (before) means the fields are changed
+ // null/empty fields in "old" (before) means the fields are not changed
+ // so we just copy the not changed fields into before
+ before.setField(f, after.getField(f));
+ }
+ }
+ before.setRowKind(RowKind.UPDATE_BEFORE);
+ after.setRowKind(RowKind.UPDATE_AFTER);
+ out.collect(before);
+ out.collect(after);
+ } else if (OP_DELETE.equals(type)) {
+ // "data" field is a row, contains deleted rows
+ RowData delete = row.getRow(0, fieldCount);
+ delete.setRowKind(RowKind.DELETE);
+ out.collect(delete);
+
+ } else {
+ if (!ignoreParseErrors) {
+ throw new IOException(format(
+ "Unknown \"type\" value \"%s\". The Maxwell JSON message is '%s'", type, new String(message)));
+ }
+ }
+ } catch (Throwable t) {
+ // a big try catch to protect the processing.
+ if (!ignoreParseErrors) {
+ throw new IOException(format(
+ "Corrupt Maxwell JSON message '%s'.", new String(message)), t);
+ }
+ }
+ }
+
+ @Override
+ public boolean isEndOfStream(RowData nextElement) {
+ return false;
+ }
+
+ @Override
+ public TypeInformation<RowData> getProducedType() {
+ return resultTypeInfo;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ MaxwellJsonDeserializationSchema that = (MaxwellJsonDeserializationSchema) o;
+ return ignoreParseErrors == that.ignoreParseErrors &&
+ fieldCount == that.fieldCount &&
+ Objects.equals(jsonDeserializer, that.jsonDeserializer) &&
+ Objects.equals(resultTypeInfo, that.resultTypeInfo);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(jsonDeserializer, resultTypeInfo, ignoreParseErrors, fieldCount);
+ }
+
+ private RowType createJsonRowType(DataType databaseSchema) {
+ // Maxwell JSON contains other information, e.g. "database", "ts"
+ // but we don't need them
+ return (RowType) DataTypes.ROW(
+ DataTypes.FIELD("data", databaseSchema),
+ DataTypes.FIELD("old", databaseSchema),
+ DataTypes.FIELD("type", DataTypes.STRING())).getLogicalType();
+ }
+}
diff --git a/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/maxwell/MaxwellJsonFormatFactory.java b/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/maxwell/MaxwellJsonFormatFactory.java
new file mode 100644
index 0000000..e3df043
--- /dev/null
+++ b/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/maxwell/MaxwellJsonFormatFactory.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.formats.json.maxwell;
+
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.api.common.serialization.SerializationSchema;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.formats.json.JsonOptions;
+import org.apache.flink.formats.json.TimestampFormat;
+import org.apache.flink.table.connector.ChangelogMode;
+import org.apache.flink.table.connector.format.DecodingFormat;
+import org.apache.flink.table.connector.format.EncodingFormat;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.source.DynamicTableSource;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.factories.DeserializationFormatFactory;
+import org.apache.flink.table.factories.DynamicTableFactory;
+import org.apache.flink.table.factories.FactoryUtil;
+import org.apache.flink.table.factories.SerializationFormatFactory;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Format factory for providing configured instances of Maxwell JSON to RowData {@link DeserializationSchema}.
+ */
+public class MaxwellJsonFormatFactory implements DeserializationFormatFactory, SerializationFormatFactory {
+
+ public static final String IDENTIFIER = "maxwell-json";
+
+ public static final ConfigOption<Boolean> IGNORE_PARSE_ERRORS = JsonOptions.IGNORE_PARSE_ERRORS;
+
+ public static final ConfigOption<String> TIMESTAMP_FORMAT = JsonOptions.TIMESTAMP_FORMAT;
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public DecodingFormat<DeserializationSchema<RowData>> createDecodingFormat(
+ DynamicTableFactory.Context context,
+ ReadableConfig formatOptions) {
+ FactoryUtil.validateFactoryOptions(this, formatOptions);
+ final boolean ignoreParseErrors = formatOptions.get(IGNORE_PARSE_ERRORS);
+ TimestampFormat timestampFormatOption = JsonOptions.getTimestampFormat(formatOptions);
+
+ return new DecodingFormat<DeserializationSchema<RowData>>() {
+ @Override
+ public DeserializationSchema<RowData> createRuntimeDecoder(
+ DynamicTableSource.Context context, DataType producedDataType) {
+ final RowType rowType = (RowType) producedDataType.getLogicalType();
+ final TypeInformation<RowData> rowDataTypeInfo =
+ (TypeInformation<RowData>) context.createTypeInformation(producedDataType);
+ return new MaxwellJsonDeserializationSchema(
+ rowType,
+ rowDataTypeInfo,
+ ignoreParseErrors,
+ timestampFormatOption);
+ }
+
+ @Override
+ public ChangelogMode getChangelogMode() {
+ return ChangelogMode.newBuilder()
+ .addContainedKind(RowKind.INSERT)
+ .addContainedKind(RowKind.UPDATE_BEFORE)
+ .addContainedKind(RowKind.UPDATE_AFTER)
+ .addContainedKind(RowKind.DELETE)
+ .build();
+ }
+ };
+ }
+
+ @Override
+ public EncodingFormat<SerializationSchema<RowData>> createEncodingFormat(
+ DynamicTableFactory.Context context,
+ ReadableConfig formatOptions) {
+ FactoryUtil.validateFactoryOptions(this, formatOptions);
+ TimestampFormat timestampFormat = JsonOptions.getTimestampFormat(formatOptions);
+
+ return new EncodingFormat<SerializationSchema<RowData>>() {
+
+ @Override
+ public ChangelogMode getChangelogMode() {
+ return ChangelogMode.newBuilder()
+ .addContainedKind(RowKind.INSERT)
+ .addContainedKind(RowKind.UPDATE_BEFORE)
+ .addContainedKind(RowKind.UPDATE_AFTER)
+ .addContainedKind(RowKind.DELETE)
+ .build();
+ }
+
+ @Override
+ public SerializationSchema<RowData> createRuntimeEncoder(
+ DynamicTableSink.Context context, DataType consumedDataType) {
+ final RowType rowType = (RowType) consumedDataType.getLogicalType();
+ return new MaxwellJsonSerializationSchema(
+ rowType,
+ timestampFormat);
+ }
+ };
+ }
+
+ @Override
+ public String factoryIdentifier() {
+ return IDENTIFIER;
+ }
+
+ @Override
+ public Set<ConfigOption<?>> requiredOptions() {
+ return Collections.emptySet();
+ }
+
+ @Override
+ public Set<ConfigOption<?>> optionalOptions() {
+ Set<ConfigOption<?>> options = new HashSet<>();
+ options.add(IGNORE_PARSE_ERRORS);
+ options.add(TIMESTAMP_FORMAT);
+ return options;
+ }
+}
diff --git a/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/maxwell/MaxwellJsonSerializationSchema.java b/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/maxwell/MaxwellJsonSerializationSchema.java
new file mode 100644
index 0000000..be9df5e
--- /dev/null
+++ b/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/maxwell/MaxwellJsonSerializationSchema.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.formats.json.maxwell;
+
+import org.apache.flink.api.common.serialization.SerializationSchema;
+import org.apache.flink.formats.json.JsonRowDataSerializationSchema;
+import org.apache.flink.formats.json.TimestampFormat;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+
+import java.util.Objects;
+
+import static org.apache.flink.table.types.utils.TypeConversions.fromLogicalToDataType;
+
+/**
+ * Serialization schema from Flink Table/SQL internal data structure {@link RowData} to maxwell-Json.
+ */
+public class MaxwellJsonSerializationSchema implements SerializationSchema<RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final StringData OP_INSERT = StringData.fromString("insert");
+ private static final StringData OP_DELETE = StringData.fromString("delete");
+
+ private final JsonRowDataSerializationSchema jsonSerializer;
+
+ /**
+ * Timestamp format specification which is used to parse timestamp.
+ */
+ private final TimestampFormat timestampFormat;
+
+ private transient GenericRowData reuse;
+
+ public MaxwellJsonSerializationSchema(RowType rowType, TimestampFormat timestampFormat) {
+ this.jsonSerializer = new JsonRowDataSerializationSchema(
+ createJsonRowType(fromLogicalToDataType(rowType)),
+ timestampFormat);
+ this.timestampFormat = timestampFormat;
+ }
+
+ @Override
+ public void open(InitializationContext context) throws Exception {
+ this.reuse = new GenericRowData(2);
+ }
+
+ @Override
+ public byte[] serialize(RowData element) {
+ reuse.setField(0, element);
+ reuse.setField(1, rowKind2String(element.getRowKind()));
+ return jsonSerializer.serialize(reuse);
+ }
+
+ private StringData rowKind2String(RowKind rowKind) {
+ switch (rowKind) {
+ case INSERT:
+ case UPDATE_AFTER:
+ return OP_INSERT;
+ case UPDATE_BEFORE:
+ case DELETE:
+ return OP_DELETE;
+ default:
+ throw new UnsupportedOperationException("Unsupported operation '" + rowKind + "' for row kind.");
+ }
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ MaxwellJsonSerializationSchema that = (MaxwellJsonSerializationSchema) o;
+ return Objects.equals(jsonSerializer, that.jsonSerializer) &&
+ timestampFormat == that.timestampFormat;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(jsonSerializer, timestampFormat);
+ }
+
+ private RowType createJsonRowType(DataType databaseSchema) {
+ DataType payload = DataTypes.ROW(
+ DataTypes.FIELD("data", databaseSchema),
+ DataTypes.FIELD("type", DataTypes.STRING()));
+ return (RowType) payload.getLogicalType();
+ }
+
+}
diff --git a/flink-formats/flink-json/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-formats/flink-json/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory
index 781a963..942b3dc 100644
--- a/flink-formats/flink-json/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory
+++ b/flink-formats/flink-json/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory
@@ -17,3 +17,4 @@ org.apache.flink.formats.json.JsonFileSystemFormatFactory
org.apache.flink.formats.json.JsonFormatFactory
org.apache.flink.formats.json.debezium.DebeziumJsonFormatFactory
org.apache.flink.formats.json.canal.CanalJsonFormatFactory
+org.apache.flink.formats.json.maxwell.MaxwellJsonFormatFactory
diff --git a/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/maxwell/MaxwellJsonFormatFactoryTest.java b/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/maxwell/MaxwellJsonFormatFactoryTest.java
new file mode 100644
index 0000000..d9dcc06
--- /dev/null
+++ b/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/maxwell/MaxwellJsonFormatFactoryTest.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.formats.json.maxwell;
+
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.api.common.serialization.SerializationSchema;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.formats.json.TimestampFormat;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.api.TableSchema;
+import org.apache.flink.table.catalog.CatalogTableImpl;
+import org.apache.flink.table.catalog.ObjectIdentifier;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.source.DynamicTableSource;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.factories.FactoryUtil;
+import org.apache.flink.table.factories.TestDynamicTableFactory;
+import org.apache.flink.table.runtime.connector.sink.SinkRuntimeProviderContext;
+import org.apache.flink.table.runtime.connector.source.ScanRuntimeProviderContext;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Consumer;
+
+import static org.apache.flink.core.testutils.FlinkMatchers.containsCause;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests for {@link MaxwellJsonFormatFactory}.
+ */
+public class MaxwellJsonFormatFactoryTest extends TestLogger {
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ private static final TableSchema SCHEMA = TableSchema.builder()
+ .field("a", DataTypes.STRING())
+ .field("b", DataTypes.INT())
+ .field("c", DataTypes.BOOLEAN())
+ .build();
+
+ private static final RowType ROW_TYPE = (RowType) SCHEMA.toRowDataType().getLogicalType();
+
+ @Test
+ public void testSeDeSchema() {
+ final MaxwellJsonDeserializationSchema expectedDeser = new MaxwellJsonDeserializationSchema(
+ ROW_TYPE,
+ InternalTypeInfo.of(ROW_TYPE),
+ true,
+ TimestampFormat.ISO_8601);
+
+ final MaxwellJsonSerializationSchema expectedSer = new MaxwellJsonSerializationSchema(
+ ROW_TYPE,
+ TimestampFormat.ISO_8601);
+
+ final Map<String, String> options = getAllOptions();
+
+ final DynamicTableSource actualSource = createTableSource(options);
+ assert actualSource instanceof TestDynamicTableFactory.DynamicTableSourceMock;
+ TestDynamicTableFactory.DynamicTableSourceMock scanSourceMock =
+ (TestDynamicTableFactory.DynamicTableSourceMock) actualSource;
+
+ DeserializationSchema<RowData> actualDeser = scanSourceMock.valueFormat
+ .createRuntimeDecoder(
+ ScanRuntimeProviderContext.INSTANCE,
+ SCHEMA.toRowDataType());
+
+ assertEquals(expectedDeser, actualDeser);
+
+ final DynamicTableSink actualSink = createTableSink(options);
+ assert actualSink instanceof TestDynamicTableFactory.DynamicTableSinkMock;
+ TestDynamicTableFactory.DynamicTableSinkMock sinkMock = (TestDynamicTableFactory.DynamicTableSinkMock) actualSink;
+
+ SerializationSchema<RowData> actualSer = sinkMock.valueFormat
+ .createRuntimeEncoder(
+ new SinkRuntimeProviderContext(false),
+ SCHEMA.toRowDataType());
+
+ assertEquals(expectedSer, actualSer);
+ }
+
+ @Test
+ public void testInvalidIgnoreParseError() {
+ thrown.expect(containsCause(new IllegalArgumentException(
+ "Unrecognized option for boolean: abc. Expected either true or false(case insensitive)")));
+
+ final Map<String, String> options =
+ getModifiedOptions(opts -> opts.put("maxwell-json.ignore-parse-errors", "abc"));
+
+ createTableSource(options);
+ }
+
+ // ------------------------------------------------------------------------
+ // Utilities
+ // ------------------------------------------------------------------------
+
+ /**
+ * Returns the full options modified by the given consumer {@code optionModifier}.
+ *
+ * @param optionModifier Consumer to modify the options
+ */
+ private Map<String, String> getModifiedOptions(Consumer<Map<String, String>> optionModifier) {
+ Map<String, String> options = getAllOptions();
+ optionModifier.accept(options);
+ return options;
+ }
+
+ private Map<String, String> getAllOptions() {
+ final Map<String, String> options = new HashMap<>();
+ options.put("connector", TestDynamicTableFactory.IDENTIFIER);
+ options.put("target", "MyTarget");
+ options.put("buffer-size", "1000");
+
+ options.put("format", "maxwell-json");
+ options.put("maxwell-json.ignore-parse-errors", "true");
+ options.put("maxwell-json.timestamp-format.standard", "ISO-8601");
+ return options;
+ }
+
+ private static DynamicTableSource createTableSource(Map<String, String> options) {
+ return FactoryUtil.createTableSource(
+ null,
+ ObjectIdentifier.of("default", "default", "t1"),
+ new CatalogTableImpl(SCHEMA, options, "mock source"),
+ new Configuration(),
+ MaxwellJsonFormatFactoryTest.class.getClassLoader());
+ }
+
+ private static DynamicTableSink createTableSink(Map<String, String> options) {
+ return FactoryUtil.createTableSink(
+ null,
+ ObjectIdentifier.of("default", "default", "t1"),
+ new CatalogTableImpl(SCHEMA, options, "mock sink"),
+ new Configuration(),
+ MaxwellJsonFormatFactoryTest.class.getClassLoader());
+ }
+}
diff --git a/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/maxwell/MaxwellJsonSerDerTest.java b/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/maxwell/MaxwellJsonSerDerTest.java
new file mode 100644
index 0000000..cab2b61
--- /dev/null
+++ b/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/maxwell/MaxwellJsonSerDerTest.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.formats.json.maxwell;
+
+import org.apache.flink.formats.json.TimestampFormat;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.util.Collector;
+
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.apache.flink.table.api.DataTypes.FIELD;
+import static org.apache.flink.table.api.DataTypes.FLOAT;
+import static org.apache.flink.table.api.DataTypes.INT;
+import static org.apache.flink.table.api.DataTypes.ROW;
+import static org.apache.flink.table.api.DataTypes.STRING;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests for {@link MaxwellJsonSerializationSchema} and {@link MaxwellJsonDeserializationSchema}.
+ */
+public class MaxwellJsonSerDerTest {
+ private static final RowType SCHEMA = (RowType) ROW(
+ FIELD("id", INT().notNull()),
+ FIELD("name", STRING()),
+ FIELD("description", STRING()),
+ FIELD("weight", FLOAT())
+ ).getLogicalType();
+
+ @Test
+ public void testSerializationDeserialization() throws Exception {
+ List<String> lines = readLines("maxwell-data.txt");
+ MaxwellJsonDeserializationSchema deserializationSchema = new MaxwellJsonDeserializationSchema(
+ SCHEMA,
+ InternalTypeInfo.of(SCHEMA),
+ false,
+ TimestampFormat.ISO_8601);
+
+ SimpleCollector collector = new SimpleCollector();
+ for (String line : lines) {
+ deserializationSchema.deserialize(line.getBytes(StandardCharsets.UTF_8), collector);
+ }
+
+ // Maxwell captures change data (`maxwell-data.txt`) on the `product` table:
+ //
+ // CREATE TABLE product (
+ // id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY,
+ // name VARCHAR(255),
+ // description VARCHAR(512),
+ // weight FLOAT
+ // );
+ // ALTER TABLE product AUTO_INCREMENT = 101;
+ //
+ // INSERT INTO product
+ // VALUES (default,"scooter","Small 2-wheel scooter",3.14),
+ // (default,"car battery","12V car battery",8.1),
+ // (default,"12-pack drill bits","12-pack of drill bits with sizes ranging from #40 to #3",0.8),
+ // (default,"hammer","12oz carpenter's hammer",0.75),
+ // (default,"hammer","14oz carpenter's hammer",0.875),
+ // (default,"hammer","16oz carpenter's hammer",1.0),
+ // (default,"rocks","box of assorted rocks",5.3),
+ // (default,"jacket","water resistent black wind breaker",0.1),
+ // (default,"spare tire","24 inch spare tire",22.2);
+ // UPDATE product SET description='18oz carpenter hammer' WHERE id=106;
+ // UPDATE product SET weight='5.1' WHERE id=107;
+ // INSERT INTO product VALUES (default,"jacket","water resistent white wind breaker",0.2);
+ // INSERT INTO product VALUES (default,"scooter","Big 2-wheel scooter ",5.18);
+ // UPDATE product SET description='new water resistent white wind breaker', weight='0.5' WHERE id=110;
+ // UPDATE product SET weight='5.17' WHERE id=111;
+ // DELETE FROM product WHERE id=111;
+ // UPDATE product SET weight='5.17' WHERE id=102 or id = 101;
+ // DELETE FROM product WHERE id=102 or id = 103;
+ List<String> expected = Arrays.asList(
+ "+I(101,scooter,Small 2-wheel scooter,3.14)",
+ "+I(102,car battery,12V car battery,8.1)",
+ "+I(103,12-pack drill bits,12-pack of drill bits with sizes ranging from #40 to #3,0.8)",
+ "+I(104,hammer,12oz carpenter's hammer,0.75)",
+ "+I(105,hammer,14oz carpenter's hammer,0.875)",
+ "+I(106,hammer,16oz carpenter's hammer,1.0)",
+ "+I(107,rocks,box of assorted rocks,5.3)",
+ "+I(108,jacket,water resistent black wind breaker,0.1)",
+ "+I(109,spare tire,24 inch spare tire,22.2)",
+ "-U(106,hammer,16oz carpenter's hammer,1.0)",
+ "+U(106,hammer,18oz carpenter hammer,1.0)",
+ "-U(107,rocks,box of assorted rocks,5.3)",
+ "+U(107,rocks,box of assorted rocks,5.1)",
+ "+I(110,jacket,water resistent white wind breaker,0.2)",
+ "+I(111,scooter,Big 2-wheel scooter ,5.18)",
+ "-U(110,jacket,water resistent white wind breaker,0.2)",
+ "+U(110,jacket,new water resistent white wind breaker,0.5)",
+ "-U(111,scooter,Big 2-wheel scooter ,5.18)",
+ "+U(111,scooter,Big 2-wheel scooter ,5.17)",
+ "-D(111,scooter,Big 2-wheel scooter ,5.17)",
+ "-U(101,scooter,Small 2-wheel scooter,3.14)",
+ "+U(101,scooter,Small 2-wheel scooter,5.17)",
+ "-U(102,car battery,12V car battery,8.1)",
+ "+U(102,car battery,12V car battery,5.17)",
+ "-D(102,car battery,12V car battery,5.17)",
+ "-D(103,12-pack drill bits,12-pack of drill bits with sizes ranging from #40 to #3,0.8)"
+ );
+ List<String> actual = collector.list.stream()
+ .map(Object::toString)
+ .collect(Collectors.toList());
+ assertEquals(expected, actual);
+
+ MaxwellJsonSerializationSchema serializationSchema = new MaxwellJsonSerializationSchema(
+ SCHEMA,
+ TimestampFormat.SQL);
+ serializationSchema.open(null);
+ List<String> result = new ArrayList<>();
+ for (RowData rowData : collector.list) {
+ result.add(new String(serializationSchema.serialize(rowData), StandardCharsets.UTF_8));
+ }
+ List<String> expectedResult = Arrays.asList(
+ "{\"data\":{\"id\":101,\"name\":\"scooter\",\"description\":\"Small 2-wheel scooter\",\"weight\":3.14},\"type\":\"insert\"}",
+ "{\"data\":{\"id\":102,\"name\":\"car battery\",\"description\":\"12V car battery\",\"weight\":8.1},\"type\":\"insert\"}",
+ "{\"data\":{\"id\":103,\"name\":\"12-pack drill bits\",\"description\":\"12-pack of drill bits with sizes ranging from #40 to #3\",\"weight\":0.8},\"type\":\"insert\"}",
+ "{\"data\":{\"id\":104,\"name\":\"hammer\",\"description\":\"12oz carpenter's hammer\",\"weight\":0.75},\"type\":\"insert\"}",
+ "{\"data\":{\"id\":105,\"name\":\"hammer\",\"description\":\"14oz carpenter's hammer\",\"weight\":0.875},\"type\":\"insert\"}",
+ "{\"data\":{\"id\":106,\"name\":\"hammer\",\"description\":\"16oz carpenter's hammer\",\"weight\":1.0},\"type\":\"insert\"}",
+ "{\"data\":{\"id\":107,\"name\":\"rocks\",\"description\":\"box of assorted rocks\",\"weight\":5.3},\"type\":\"insert\"}",
+ "{\"data\":{\"id\":108,\"name\":\"jacket\",\"description\":\"water resistent black wind breaker\",\"weight\":0.1},\"type\":\"insert\"}",
+ "{\"data\":{\"id\":109,\"name\":\"spare tire\",\"description\":\"24 inch spare tire\",\"weight\":22.2},\"type\":\"insert\"}",
+ "{\"data\":{\"id\":106,\"name\":\"hammer\",\"description\":\"16oz carpenter's hammer\",\"weight\":1.0},\"type\":\"delete\"}",
+ "{\"data\":{\"id\":106,\"name\":\"hammer\",\"description\":\"18oz carpenter hammer\",\"weight\":1.0},\"type\":\"insert\"}",
+ "{\"data\":{\"id\":107,\"name\":\"rocks\",\"description\":\"box of assorted rocks\",\"weight\":5.3},\"type\":\"delete\"}",
+ "{\"data\":{\"id\":107,\"name\":\"rocks\",\"description\":\"box of assorted rocks\",\"weight\":5.1},\"type\":\"insert\"}",
+ "{\"data\":{\"id\":110,\"name\":\"jacket\",\"description\":\"water resistent white wind breaker\",\"weight\":0.2},\"type\":\"insert\"}",
+ "{\"data\":{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.18},\"type\":\"insert\"}",
+ "{\"data\":{\"id\":110,\"name\":\"jacket\",\"description\":\"water resistent white wind breaker\",\"weight\":0.2},\"type\":\"delete\"}",
+ "{\"data\":{\"id\":110,\"name\":\"jacket\",\"description\":\"new water resistent white wind breaker\",\"weight\":0.5},\"type\":\"insert\"}",
+ "{\"data\":{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.18},\"type\":\"delete\"}",
+ "{\"data\":{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.17},\"type\":\"insert\"}",
+ "{\"data\":{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.17},\"type\":\"delete\"}",
+ "{\"data\":{\"id\":101,\"name\":\"scooter\",\"description\":\"Small 2-wheel scooter\",\"weight\":3.14},\"type\":\"delete\"}",
+ "{\"data\":{\"id\":101,\"name\":\"scooter\",\"description\":\"Small 2-wheel scooter\",\"weight\":5.17},\"type\":\"insert\"}",
+ "{\"data\":{\"id\":102,\"name\":\"car battery\",\"description\":\"12V car battery\",\"weight\":8.1},\"type\":\"delete\"}",
+ "{\"data\":{\"id\":102,\"name\":\"car battery\",\"description\":\"12V car battery\",\"weight\":5.17},\"type\":\"insert\"}",
+ "{\"data\":{\"id\":102,\"name\":\"car battery\",\"description\":\"12V car battery\",\"weight\":5.17},\"type\":\"delete\"}",
+ "{\"data\":{\"id\":103,\"name\":\"12-pack drill bits\",\"description\":\"12-pack of drill bits with sizes ranging from #40 to #3\",\"weight\":0.8},\"type\":\"delete\"}"
+ );
+ assertEquals(expectedResult, result);
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // Utilities
+ // --------------------------------------------------------------------------------------------
+
+ private static List<String> readLines(String resource) throws IOException {
+ final URL url = MaxwellJsonSerDerTest.class.getClassLoader().getResource(resource);
+ assert url != null;
+ Path path = new File(url.getFile()).toPath();
+ return Files.readAllLines(path);
+ }
+
+ private static class SimpleCollector implements Collector<RowData> {
+
+ private List<RowData> list = new ArrayList<>();
+
+ @Override
+ public void collect(RowData record) {
+ list.add(record);
+ }
+
+ @Override
+ public void close() {
+ // do nothing
+ }
+ }
+}
diff --git a/flink-formats/flink-json/src/test/resources/maxwell-data.txt b/flink-formats/flink-json/src/test/resources/maxwell-data.txt
new file mode 100644
index 0000000..ecba573
--- /dev/null
+++ b/flink-formats/flink-json/src/test/resources/maxwell-data.txt
@@ -0,0 +1,20 @@
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":0,"data":{"id":101,"name":"scooter","description":"Small 2-wheel scooter","weight":3.14}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":1,"data":{"id":102,"name":"car battery","description":"12V car battery","weight":8.1}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":2,"data":{"id":103,"name":"12-pack drill bits","description":"12-pack of drill bits with sizes ranging from #40 to #3","weight":0.8}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":3,"data":{"id":104,"name":"hammer","description":"12oz carpenter's hammer","weight":0.75}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":4,"data":{"id":105,"name":"hammer","description":"14oz carpenter's hammer","weight":0.875}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":5,"data":{"id":106,"name":"hammer","description":"16oz carpenter's hammer","weight":1.0}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":6,"data":{"id":107,"name":"rocks","description":"box of assorted rocks","weight":5.3}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":7,"data":{"id":108,"name":"jacket","description":"water resistent black wind breaker","weight":0.1}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"commit":true,"data":{"id":109,"name":"spare tire","description":"24 inch spare tire","weight":22.2}}
+{"database":"test","table":"product","type":"update","ts":1596684893,"xid":7152,"commit":true,"data":{"id":106,"name":"hammer","description":"18oz carpenter hammer","weight":1.0},"old":{"description":"16oz carpenter's hammer"}}
+{"database":"test","table":"product","type":"update","ts":1596684897,"xid":7169,"commit":true,"data":{"id":107,"name":"rocks","description":"box of assorted rocks","weight":5.1},"old":{"weight":5.3}}
+{"database":"test","table":"product","type":"insert","ts":1596684900,"xid":7186,"commit":true,"data":{"id":110,"name":"jacket","description":"water resistent white wind breaker","weight":0.2}}
+{"database":"test","table":"product","type":"insert","ts":1596684904,"xid":7201,"commit":true,"data":{"id":111,"name":"scooter","description":"Big 2-wheel scooter ","weight":5.18}}
+{"database":"test","table":"product","type":"update","ts":1596684906,"xid":7216,"commit":true,"data":{"id":110,"name":"jacket","description":"new water resistent white wind breaker","weight":0.5},"old":{"description":"water resistent white wind breaker","weight":0.2}}
+{"database":"test","table":"product","type":"update","ts":1596684912,"xid":7235,"commit":true,"data":{"id":111,"name":"scooter","description":"Big 2-wheel scooter ","weight":5.17},"old":{"weight":5.18}}
+{"database":"test","table":"product","type":"delete","ts":1596684914,"xid":7250,"commit":true,"data":{"id":111,"name":"scooter","description":"Big 2-wheel scooter ","weight":5.17}}
+{"database":"test","table":"product","type":"update","ts":1596684928,"xid":7291,"xoffset":0,"data":{"id":101,"name":"scooter","description":"Small 2-wheel scooter","weight":5.17},"old":{"weight":3.14}}
+{"database":"test","table":"product","type":"update","ts":1596684928,"xid":7291,"commit":true,"data":{"id":102,"name":"car battery","description":"12V car battery","weight":5.17},"old":{"weight":8.1}}
+{"database":"test","table":"product","type":"delete","ts":1596684938,"xid":7322,"xoffset":0,"data":{"id":102,"name":"car battery","description":"12V car battery","weight":5.17}}
+{"database":"test","table":"product","type":"delete","ts":1596684938,"xid":7322,"commit":true,"data":{"id":103,"name":"12-pack drill bits","description":"12-pack of drill bits with sizes ranging from #40 to #3","weight":0.8}}