You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ja...@apache.org on 2020/08/12 03:13:22 UTC

[flink] branch master updated: [FLINK-18844][json][maxwell] Support maxwell-json format to read Maxwell changelogs

This is an automated email from the ASF dual-hosted git repository.

jark pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/master by this push:
     new 2d75031  [FLINK-18844][json][maxwell] Support maxwell-json format to read Maxwell changelogs
2d75031 is described below

commit 2d75031d3d6bdc11c7b3fcf25cd43f396677003c
Author: dijie <nj...@gmail.com>
AuthorDate: Wed Aug 12 11:11:58 2020 +0800

    [FLINK-18844][json][maxwell] Support maxwell-json format to read Maxwell changelogs
    
    This closes #13090
---
 .../maxwell/MaxwellJsonDeserializationSchema.java  | 179 +++++++++++++++++++
 .../json/maxwell/MaxwellJsonFormatFactory.java     | 139 +++++++++++++++
 .../maxwell/MaxwellJsonSerializationSchema.java    | 111 ++++++++++++
 .../org.apache.flink.table.factories.Factory       |   1 +
 .../json/maxwell/MaxwellJsonFormatFactoryTest.java | 159 +++++++++++++++++
 .../json/maxwell/MaxwellJsonSerDerTest.java        | 198 +++++++++++++++++++++
 .../flink-json/src/test/resources/maxwell-data.txt |  20 +++
 7 files changed, 807 insertions(+)

diff --git a/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/maxwell/MaxwellJsonDeserializationSchema.java b/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/maxwell/MaxwellJsonDeserializationSchema.java
new file mode 100644
index 0000000..c34ab55
--- /dev/null
+++ b/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/maxwell/MaxwellJsonDeserializationSchema.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.formats.json.maxwell;
+
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.formats.json.JsonRowDataDeserializationSchema;
+import org.apache.flink.formats.json.TimestampFormat;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static java.lang.String.format;
+import static org.apache.flink.table.types.utils.TypeConversions.fromLogicalToDataType;
+
+/**
+ * Deserialization schema from Maxwell JSON to Flink Table/SQL internal data structure {@link RowData}.
+ * The deserialization schema knows Maxwell's schema definition and can extract the database data
+ * and convert into {@link RowData} with {@link RowKind}.
+ *
+ * <p>Deserializes a <code>byte[]</code> message as a JSON object and reads
+ * the specified fields.
+ *
+ * <p>Failures during deserialization are forwarded as wrapped IOExceptions.
+ *
+ * @see <a href="http://maxwells-daemon.io/">Maxwell</a>
+ */
+public class MaxwellJsonDeserializationSchema implements DeserializationSchema<RowData> {
+	private static final long serialVersionUID = 1L;
+
+	private static final String OP_INSERT = "insert";
+	private static final String OP_UPDATE = "update";
+	private static final String OP_DELETE = "delete";
+
+	/** The deserializer to deserialize Maxwell JSON data. */
+	private final JsonRowDataDeserializationSchema jsonDeserializer;
+
+	/** TypeInformation of the produced {@link RowData}. **/
+	private final TypeInformation<RowData> resultTypeInfo;
+
+	/** Flag indicating whether to ignore invalid fields/rows (default: throw an exception). */
+	private final boolean ignoreParseErrors;
+
+	/** Number of fields. */
+	private final int fieldCount;
+
+	public MaxwellJsonDeserializationSchema(RowType rowType,
+			TypeInformation<RowData> resultTypeInfo,
+			boolean ignoreParseErrors,
+			TimestampFormat timestampFormatOption) {
+		this.resultTypeInfo = resultTypeInfo;
+		this.ignoreParseErrors = ignoreParseErrors;
+		this.fieldCount = rowType.getFieldCount();
+		this.jsonDeserializer = new JsonRowDataDeserializationSchema(
+			createJsonRowType(fromLogicalToDataType(rowType)),
+			// the result type is never used, so it's fine to pass in Canal's result type
+			resultTypeInfo,
+			false, // ignoreParseErrors already contains the functionality of failOnMissingField
+			ignoreParseErrors,
+			timestampFormatOption);
+	}
+
+	@Override
+	public RowData deserialize(byte[] message) throws IOException {
+		throw new RuntimeException(
+			"Please invoke DeserializationSchema#deserialize(byte[], Collector<RowData>) instead.");
+	}
+
+	@Override
+	public void deserialize(byte[] message, Collector<RowData> out) throws IOException {
+		try {
+			RowData row = jsonDeserializer.deserialize(message);
+			String type = row.getString(2).toString(); // "type" field
+			if (OP_INSERT.equals(type)) {
+				// "data" field is a row, contains inserted rows
+				RowData insert = row.getRow(0, fieldCount);
+				insert.setRowKind(RowKind.INSERT);
+				out.collect(insert);
+			} else if (OP_UPDATE.equals(type)) {
+				// "data" field is a row, contains new rows
+				// "old" field is an array of row, contains old values
+				// the underlying JSON deserialization schema always produce GenericRowData.
+				GenericRowData after = (GenericRowData) row.getRow(0, fieldCount); // "data" field
+				GenericRowData before = (GenericRowData) row.getRow(1, fieldCount); // "old" field
+				for (int f = 0; f < fieldCount; f++) {
+					if (before.isNullAt(f)) {
+						// not null fields in "old" (before) means the fields are changed
+						// null/empty fields in "old" (before) means the fields are not changed
+						// so we just copy the not changed fields into before
+						before.setField(f, after.getField(f));
+					}
+				}
+				before.setRowKind(RowKind.UPDATE_BEFORE);
+				after.setRowKind(RowKind.UPDATE_AFTER);
+				out.collect(before);
+				out.collect(after);
+			} else if (OP_DELETE.equals(type)) {
+				// "data" field is a row, contains deleted rows
+				RowData delete = row.getRow(0, fieldCount);
+				delete.setRowKind(RowKind.DELETE);
+				out.collect(delete);
+
+			} else {
+				if (!ignoreParseErrors) {
+					throw new IOException(format(
+						"Unknown \"type\" value \"%s\". The Maxwell JSON message is '%s'", type, new String(message)));
+				}
+			}
+		} catch (Throwable t) {
+			// a big try catch to protect the processing.
+			if (!ignoreParseErrors) {
+				throw new IOException(format(
+					"Corrupt Maxwell JSON message '%s'.", new String(message)), t);
+			}
+		}
+	}
+
+	@Override
+	public boolean isEndOfStream(RowData nextElement) {
+		return false;
+	}
+
+	@Override
+	public TypeInformation<RowData> getProducedType() {
+		return resultTypeInfo;
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o) {
+			return true;
+		}
+		if (o == null || getClass() != o.getClass()) {
+			return false;
+		}
+		MaxwellJsonDeserializationSchema that = (MaxwellJsonDeserializationSchema) o;
+		return ignoreParseErrors == that.ignoreParseErrors &&
+			fieldCount == that.fieldCount &&
+			Objects.equals(jsonDeserializer, that.jsonDeserializer) &&
+			Objects.equals(resultTypeInfo, that.resultTypeInfo);
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(jsonDeserializer, resultTypeInfo, ignoreParseErrors, fieldCount);
+	}
+
+	private RowType createJsonRowType(DataType databaseSchema) {
+		// Maxwell JSON contains other information, e.g. "database", "ts"
+		// but we don't need them
+		return (RowType) DataTypes.ROW(
+			DataTypes.FIELD("data", databaseSchema),
+			DataTypes.FIELD("old", databaseSchema),
+			DataTypes.FIELD("type", DataTypes.STRING())).getLogicalType();
+	}
+}
diff --git a/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/maxwell/MaxwellJsonFormatFactory.java b/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/maxwell/MaxwellJsonFormatFactory.java
new file mode 100644
index 0000000..e3df043
--- /dev/null
+++ b/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/maxwell/MaxwellJsonFormatFactory.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.formats.json.maxwell;
+
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.api.common.serialization.SerializationSchema;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.formats.json.JsonOptions;
+import org.apache.flink.formats.json.TimestampFormat;
+import org.apache.flink.table.connector.ChangelogMode;
+import org.apache.flink.table.connector.format.DecodingFormat;
+import org.apache.flink.table.connector.format.EncodingFormat;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.source.DynamicTableSource;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.factories.DeserializationFormatFactory;
+import org.apache.flink.table.factories.DynamicTableFactory;
+import org.apache.flink.table.factories.FactoryUtil;
+import org.apache.flink.table.factories.SerializationFormatFactory;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Format factory for providing configured instances of Maxwell JSON to RowData {@link DeserializationSchema}.
+ */
+public class MaxwellJsonFormatFactory implements DeserializationFormatFactory, SerializationFormatFactory {
+
+	public static final String IDENTIFIER = "maxwell-json";
+
+	public static final ConfigOption<Boolean> IGNORE_PARSE_ERRORS = JsonOptions.IGNORE_PARSE_ERRORS;
+
+	public static final ConfigOption<String> TIMESTAMP_FORMAT = JsonOptions.TIMESTAMP_FORMAT;
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public DecodingFormat<DeserializationSchema<RowData>> createDecodingFormat(
+			DynamicTableFactory.Context context,
+			ReadableConfig formatOptions) {
+		FactoryUtil.validateFactoryOptions(this, formatOptions);
+		final boolean ignoreParseErrors = formatOptions.get(IGNORE_PARSE_ERRORS);
+		TimestampFormat timestampFormatOption = JsonOptions.getTimestampFormat(formatOptions);
+
+		return new DecodingFormat<DeserializationSchema<RowData>>() {
+			@Override
+			public DeserializationSchema<RowData> createRuntimeDecoder(
+					DynamicTableSource.Context context, DataType producedDataType) {
+				final RowType rowType = (RowType) producedDataType.getLogicalType();
+				final TypeInformation<RowData> rowDataTypeInfo =
+					(TypeInformation<RowData>) context.createTypeInformation(producedDataType);
+				return new MaxwellJsonDeserializationSchema(
+					rowType,
+					rowDataTypeInfo,
+					ignoreParseErrors,
+					timestampFormatOption);
+			}
+
+			@Override
+			public ChangelogMode getChangelogMode() {
+				return ChangelogMode.newBuilder()
+					.addContainedKind(RowKind.INSERT)
+					.addContainedKind(RowKind.UPDATE_BEFORE)
+					.addContainedKind(RowKind.UPDATE_AFTER)
+					.addContainedKind(RowKind.DELETE)
+					.build();
+			}
+		};
+	}
+
+	@Override
+	public EncodingFormat<SerializationSchema<RowData>> createEncodingFormat(
+			DynamicTableFactory.Context context,
+			ReadableConfig formatOptions) {
+		FactoryUtil.validateFactoryOptions(this, formatOptions);
+		TimestampFormat timestampFormat = JsonOptions.getTimestampFormat(formatOptions);
+
+		return new EncodingFormat<SerializationSchema<RowData>>() {
+
+			@Override
+			public ChangelogMode getChangelogMode() {
+				return ChangelogMode.newBuilder()
+					.addContainedKind(RowKind.INSERT)
+					.addContainedKind(RowKind.UPDATE_BEFORE)
+					.addContainedKind(RowKind.UPDATE_AFTER)
+					.addContainedKind(RowKind.DELETE)
+					.build();
+			}
+
+			@Override
+			public SerializationSchema<RowData> createRuntimeEncoder(
+					DynamicTableSink.Context context, DataType consumedDataType) {
+				final RowType rowType = (RowType) consumedDataType.getLogicalType();
+				return new MaxwellJsonSerializationSchema(
+					rowType,
+					timestampFormat);
+			}
+		};
+	}
+
+	@Override
+	public String factoryIdentifier() {
+		return IDENTIFIER;
+	}
+
+	@Override
+	public Set<ConfigOption<?>> requiredOptions() {
+		return Collections.emptySet();
+	}
+
+	@Override
+	public Set<ConfigOption<?>> optionalOptions() {
+		Set<ConfigOption<?>> options = new HashSet<>();
+		options.add(IGNORE_PARSE_ERRORS);
+		options.add(TIMESTAMP_FORMAT);
+		return options;
+	}
+}
diff --git a/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/maxwell/MaxwellJsonSerializationSchema.java b/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/maxwell/MaxwellJsonSerializationSchema.java
new file mode 100644
index 0000000..be9df5e
--- /dev/null
+++ b/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/maxwell/MaxwellJsonSerializationSchema.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.formats.json.maxwell;
+
+import org.apache.flink.api.common.serialization.SerializationSchema;
+import org.apache.flink.formats.json.JsonRowDataSerializationSchema;
+import org.apache.flink.formats.json.TimestampFormat;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+
+import java.util.Objects;
+
+import static org.apache.flink.table.types.utils.TypeConversions.fromLogicalToDataType;
+
+/**
+ * Serialization schema from Flink Table/SQL internal data structure {@link RowData} to maxwell-Json.
+ */
+public class MaxwellJsonSerializationSchema implements SerializationSchema<RowData> {
+	private static final long serialVersionUID = 1L;
+
+	private static final StringData OP_INSERT = StringData.fromString("insert");
+	private static final StringData OP_DELETE = StringData.fromString("delete");
+
+	private final JsonRowDataSerializationSchema jsonSerializer;
+
+	/**
+	 * Timestamp format specification which is used to parse timestamp.
+	 */
+	private final TimestampFormat timestampFormat;
+
+	private transient GenericRowData reuse;
+
+	public MaxwellJsonSerializationSchema(RowType rowType, TimestampFormat timestampFormat) {
+		this.jsonSerializer = new JsonRowDataSerializationSchema(
+			createJsonRowType(fromLogicalToDataType(rowType)),
+			timestampFormat);
+		this.timestampFormat = timestampFormat;
+	}
+
+	@Override
+	public void open(InitializationContext context) throws Exception {
+		this.reuse = new GenericRowData(2);
+	}
+
+	@Override
+	public byte[] serialize(RowData element) {
+		reuse.setField(0, element);
+		reuse.setField(1, rowKind2String(element.getRowKind()));
+		return jsonSerializer.serialize(reuse);
+	}
+
+	private StringData rowKind2String(RowKind rowKind) {
+		switch (rowKind) {
+			case INSERT:
+			case UPDATE_AFTER:
+				return OP_INSERT;
+			case UPDATE_BEFORE:
+			case DELETE:
+				return OP_DELETE;
+			default:
+				throw new UnsupportedOperationException("Unsupported operation '" + rowKind + "' for row kind.");
+		}
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o) {
+			return true;
+		}
+		if (o == null || getClass() != o.getClass()) {
+			return false;
+		}
+		MaxwellJsonSerializationSchema that = (MaxwellJsonSerializationSchema) o;
+		return Objects.equals(jsonSerializer, that.jsonSerializer) &&
+			timestampFormat == that.timestampFormat;
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(jsonSerializer, timestampFormat);
+	}
+
+	private RowType createJsonRowType(DataType databaseSchema) {
+		DataType payload = DataTypes.ROW(
+			DataTypes.FIELD("data", databaseSchema),
+			DataTypes.FIELD("type", DataTypes.STRING()));
+		return (RowType) payload.getLogicalType();
+	}
+
+}
diff --git a/flink-formats/flink-json/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-formats/flink-json/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory
index 781a963..942b3dc 100644
--- a/flink-formats/flink-json/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory
+++ b/flink-formats/flink-json/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory
@@ -17,3 +17,4 @@ org.apache.flink.formats.json.JsonFileSystemFormatFactory
 org.apache.flink.formats.json.JsonFormatFactory
 org.apache.flink.formats.json.debezium.DebeziumJsonFormatFactory
 org.apache.flink.formats.json.canal.CanalJsonFormatFactory
+org.apache.flink.formats.json.maxwell.MaxwellJsonFormatFactory
diff --git a/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/maxwell/MaxwellJsonFormatFactoryTest.java b/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/maxwell/MaxwellJsonFormatFactoryTest.java
new file mode 100644
index 0000000..d9dcc06
--- /dev/null
+++ b/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/maxwell/MaxwellJsonFormatFactoryTest.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.formats.json.maxwell;
+
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.api.common.serialization.SerializationSchema;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.formats.json.TimestampFormat;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.api.TableSchema;
+import org.apache.flink.table.catalog.CatalogTableImpl;
+import org.apache.flink.table.catalog.ObjectIdentifier;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.source.DynamicTableSource;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.factories.FactoryUtil;
+import org.apache.flink.table.factories.TestDynamicTableFactory;
+import org.apache.flink.table.runtime.connector.sink.SinkRuntimeProviderContext;
+import org.apache.flink.table.runtime.connector.source.ScanRuntimeProviderContext;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Consumer;
+
+import static org.apache.flink.core.testutils.FlinkMatchers.containsCause;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests for {@link MaxwellJsonFormatFactory}.
+ */
+public class MaxwellJsonFormatFactoryTest extends TestLogger {
+	@Rule
+	public ExpectedException thrown = ExpectedException.none();
+
+	private static final TableSchema SCHEMA = TableSchema.builder()
+		.field("a", DataTypes.STRING())
+		.field("b", DataTypes.INT())
+		.field("c", DataTypes.BOOLEAN())
+		.build();
+
+	private static final RowType ROW_TYPE = (RowType) SCHEMA.toRowDataType().getLogicalType();
+
+	@Test
+	public void testSeDeSchema() {
+		final MaxwellJsonDeserializationSchema expectedDeser = new MaxwellJsonDeserializationSchema(
+			ROW_TYPE,
+			InternalTypeInfo.of(ROW_TYPE),
+			true,
+			TimestampFormat.ISO_8601);
+
+		final MaxwellJsonSerializationSchema expectedSer = new MaxwellJsonSerializationSchema(
+			ROW_TYPE,
+			TimestampFormat.ISO_8601);
+
+		final Map<String, String> options = getAllOptions();
+
+		final DynamicTableSource actualSource = createTableSource(options);
+		assert actualSource instanceof TestDynamicTableFactory.DynamicTableSourceMock;
+		TestDynamicTableFactory.DynamicTableSourceMock scanSourceMock =
+			(TestDynamicTableFactory.DynamicTableSourceMock) actualSource;
+
+		DeserializationSchema<RowData> actualDeser = scanSourceMock.valueFormat
+			.createRuntimeDecoder(
+				ScanRuntimeProviderContext.INSTANCE,
+				SCHEMA.toRowDataType());
+
+		assertEquals(expectedDeser, actualDeser);
+
+		final DynamicTableSink actualSink = createTableSink(options);
+		assert actualSink instanceof TestDynamicTableFactory.DynamicTableSinkMock;
+		TestDynamicTableFactory.DynamicTableSinkMock sinkMock = (TestDynamicTableFactory.DynamicTableSinkMock) actualSink;
+
+		SerializationSchema<RowData> actualSer = sinkMock.valueFormat
+			.createRuntimeEncoder(
+				new SinkRuntimeProviderContext(false),
+				SCHEMA.toRowDataType());
+
+		assertEquals(expectedSer, actualSer);
+	}
+
+	@Test
+	public void testInvalidIgnoreParseError() {
+		thrown.expect(containsCause(new IllegalArgumentException(
+			"Unrecognized option for boolean: abc. Expected either true or false(case insensitive)")));
+
+		final Map<String, String> options =
+			getModifiedOptions(opts -> opts.put("maxwell-json.ignore-parse-errors", "abc"));
+
+		createTableSource(options);
+	}
+
+	// ------------------------------------------------------------------------
+	//  Utilities
+	// ------------------------------------------------------------------------
+
+	/**
+	 * Returns the full options modified by the given consumer {@code optionModifier}.
+	 *
+	 * @param optionModifier Consumer to modify the options
+	 */
+	private Map<String, String> getModifiedOptions(Consumer<Map<String, String>> optionModifier) {
+		Map<String, String> options = getAllOptions();
+		optionModifier.accept(options);
+		return options;
+	}
+
+	private Map<String, String> getAllOptions() {
+		final Map<String, String> options = new HashMap<>();
+		options.put("connector", TestDynamicTableFactory.IDENTIFIER);
+		options.put("target", "MyTarget");
+		options.put("buffer-size", "1000");
+
+		options.put("format", "maxwell-json");
+		options.put("maxwell-json.ignore-parse-errors", "true");
+		options.put("maxwell-json.timestamp-format.standard", "ISO-8601");
+		return options;
+	}
+
+	private static DynamicTableSource createTableSource(Map<String, String> options) {
+		return FactoryUtil.createTableSource(
+			null,
+			ObjectIdentifier.of("default", "default", "t1"),
+			new CatalogTableImpl(SCHEMA, options, "mock source"),
+			new Configuration(),
+			MaxwellJsonFormatFactoryTest.class.getClassLoader());
+	}
+
+	private static DynamicTableSink createTableSink(Map<String, String> options) {
+		return FactoryUtil.createTableSink(
+			null,
+			ObjectIdentifier.of("default", "default", "t1"),
+			new CatalogTableImpl(SCHEMA, options, "mock sink"),
+			new Configuration(),
+			MaxwellJsonFormatFactoryTest.class.getClassLoader());
+	}
+}
diff --git a/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/maxwell/MaxwellJsonSerDerTest.java b/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/maxwell/MaxwellJsonSerDerTest.java
new file mode 100644
index 0000000..cab2b61
--- /dev/null
+++ b/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/maxwell/MaxwellJsonSerDerTest.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.formats.json.maxwell;
+
+import org.apache.flink.formats.json.TimestampFormat;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.util.Collector;
+
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.apache.flink.table.api.DataTypes.FIELD;
+import static org.apache.flink.table.api.DataTypes.FLOAT;
+import static org.apache.flink.table.api.DataTypes.INT;
+import static org.apache.flink.table.api.DataTypes.ROW;
+import static org.apache.flink.table.api.DataTypes.STRING;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests for {@link MaxwellJsonSerializationSchema} and {@link MaxwellJsonDeserializationSchema}.
+ */
+public class MaxwellJsonSerDerTest {
+	private static final RowType SCHEMA = (RowType) ROW(
+		FIELD("id", INT().notNull()),
+		FIELD("name", STRING()),
+		FIELD("description", STRING()),
+		FIELD("weight", FLOAT())
+	).getLogicalType();
+
+	@Test
+	public void testSerializationDeserialization() throws Exception {
+		List<String> lines = readLines("maxwell-data.txt");
+		MaxwellJsonDeserializationSchema deserializationSchema = new MaxwellJsonDeserializationSchema(
+			SCHEMA,
+			InternalTypeInfo.of(SCHEMA),
+			false,
+			TimestampFormat.ISO_8601);
+
+		SimpleCollector collector = new SimpleCollector();
+		for (String line : lines) {
+			deserializationSchema.deserialize(line.getBytes(StandardCharsets.UTF_8), collector);
+		}
+
+		// Maxwell captures change data (`maxwell-data.txt`) on the `product` table:
+		//
+		// CREATE TABLE product (
+		//  id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY,
+		//  name VARCHAR(255),
+		//  description VARCHAR(512),
+		//  weight FLOAT
+		// );
+		// ALTER TABLE product AUTO_INCREMENT = 101;
+		//
+		// INSERT INTO product
+		// VALUES (default,"scooter","Small 2-wheel scooter",3.14),
+		//        (default,"car battery","12V car battery",8.1),
+		//        (default,"12-pack drill bits","12-pack of drill bits with sizes ranging from #40 to #3",0.8),
+		//        (default,"hammer","12oz carpenter's hammer",0.75),
+		//        (default,"hammer","14oz carpenter's hammer",0.875),
+		//        (default,"hammer","16oz carpenter's hammer",1.0),
+		//        (default,"rocks","box of assorted rocks",5.3),
+		//        (default,"jacket","water resistent black wind breaker",0.1),
+		//        (default,"spare tire","24 inch spare tire",22.2);
+		// UPDATE product SET description='18oz carpenter hammer' WHERE id=106;
+		// UPDATE product SET weight='5.1' WHERE id=107;
+		// INSERT INTO product VALUES (default,"jacket","water resistent white wind breaker",0.2);
+		// INSERT INTO product VALUES (default,"scooter","Big 2-wheel scooter ",5.18);
+		// UPDATE product SET description='new water resistent white wind breaker', weight='0.5' WHERE id=110;
+		// UPDATE product SET weight='5.17' WHERE id=111;
+		// DELETE FROM product WHERE id=111;
+		// UPDATE product SET weight='5.17' WHERE id=102 or id = 101;
+		// DELETE FROM product WHERE id=102 or id = 103;
+		List<String> expected = Arrays.asList(
+			"+I(101,scooter,Small 2-wheel scooter,3.14)",
+			"+I(102,car battery,12V car battery,8.1)",
+			"+I(103,12-pack drill bits,12-pack of drill bits with sizes ranging from #40 to #3,0.8)",
+			"+I(104,hammer,12oz carpenter's hammer,0.75)",
+			"+I(105,hammer,14oz carpenter's hammer,0.875)",
+			"+I(106,hammer,16oz carpenter's hammer,1.0)",
+			"+I(107,rocks,box of assorted rocks,5.3)",
+			"+I(108,jacket,water resistent black wind breaker,0.1)",
+			"+I(109,spare tire,24 inch spare tire,22.2)",
+			"-U(106,hammer,16oz carpenter's hammer,1.0)",
+			"+U(106,hammer,18oz carpenter hammer,1.0)",
+			"-U(107,rocks,box of assorted rocks,5.3)",
+			"+U(107,rocks,box of assorted rocks,5.1)",
+			"+I(110,jacket,water resistent white wind breaker,0.2)",
+			"+I(111,scooter,Big 2-wheel scooter ,5.18)",
+			"-U(110,jacket,water resistent white wind breaker,0.2)",
+			"+U(110,jacket,new water resistent white wind breaker,0.5)",
+			"-U(111,scooter,Big 2-wheel scooter ,5.18)",
+			"+U(111,scooter,Big 2-wheel scooter ,5.17)",
+			"-D(111,scooter,Big 2-wheel scooter ,5.17)",
+			"-U(101,scooter,Small 2-wheel scooter,3.14)",
+			"+U(101,scooter,Small 2-wheel scooter,5.17)",
+			"-U(102,car battery,12V car battery,8.1)",
+			"+U(102,car battery,12V car battery,5.17)",
+			"-D(102,car battery,12V car battery,5.17)",
+			"-D(103,12-pack drill bits,12-pack of drill bits with sizes ranging from #40 to #3,0.8)"
+		);
+		List<String> actual = collector.list.stream()
+			.map(Object::toString)
+			.collect(Collectors.toList());
+		assertEquals(expected, actual);
+
+		MaxwellJsonSerializationSchema serializationSchema = new MaxwellJsonSerializationSchema(
+			SCHEMA,
+			TimestampFormat.SQL);
+		serializationSchema.open(null);
+		List<String> result = new ArrayList<>();
+		for (RowData rowData : collector.list) {
+			result.add(new String(serializationSchema.serialize(rowData), StandardCharsets.UTF_8));
+		}
+		List<String> expectedResult = Arrays.asList(
+			"{\"data\":{\"id\":101,\"name\":\"scooter\",\"description\":\"Small 2-wheel scooter\",\"weight\":3.14},\"type\":\"insert\"}",
+			"{\"data\":{\"id\":102,\"name\":\"car battery\",\"description\":\"12V car battery\",\"weight\":8.1},\"type\":\"insert\"}",
+			"{\"data\":{\"id\":103,\"name\":\"12-pack drill bits\",\"description\":\"12-pack of drill bits with sizes ranging from #40 to #3\",\"weight\":0.8},\"type\":\"insert\"}",
+			"{\"data\":{\"id\":104,\"name\":\"hammer\",\"description\":\"12oz carpenter's hammer\",\"weight\":0.75},\"type\":\"insert\"}",
+			"{\"data\":{\"id\":105,\"name\":\"hammer\",\"description\":\"14oz carpenter's hammer\",\"weight\":0.875},\"type\":\"insert\"}",
+			"{\"data\":{\"id\":106,\"name\":\"hammer\",\"description\":\"16oz carpenter's hammer\",\"weight\":1.0},\"type\":\"insert\"}",
+			"{\"data\":{\"id\":107,\"name\":\"rocks\",\"description\":\"box of assorted rocks\",\"weight\":5.3},\"type\":\"insert\"}",
+			"{\"data\":{\"id\":108,\"name\":\"jacket\",\"description\":\"water resistent black wind breaker\",\"weight\":0.1},\"type\":\"insert\"}",
+			"{\"data\":{\"id\":109,\"name\":\"spare tire\",\"description\":\"24 inch spare tire\",\"weight\":22.2},\"type\":\"insert\"}",
+			"{\"data\":{\"id\":106,\"name\":\"hammer\",\"description\":\"16oz carpenter's hammer\",\"weight\":1.0},\"type\":\"delete\"}",
+			"{\"data\":{\"id\":106,\"name\":\"hammer\",\"description\":\"18oz carpenter hammer\",\"weight\":1.0},\"type\":\"insert\"}",
+			"{\"data\":{\"id\":107,\"name\":\"rocks\",\"description\":\"box of assorted rocks\",\"weight\":5.3},\"type\":\"delete\"}",
+			"{\"data\":{\"id\":107,\"name\":\"rocks\",\"description\":\"box of assorted rocks\",\"weight\":5.1},\"type\":\"insert\"}",
+			"{\"data\":{\"id\":110,\"name\":\"jacket\",\"description\":\"water resistent white wind breaker\",\"weight\":0.2},\"type\":\"insert\"}",
+			"{\"data\":{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.18},\"type\":\"insert\"}",
+			"{\"data\":{\"id\":110,\"name\":\"jacket\",\"description\":\"water resistent white wind breaker\",\"weight\":0.2},\"type\":\"delete\"}",
+			"{\"data\":{\"id\":110,\"name\":\"jacket\",\"description\":\"new water resistent white wind breaker\",\"weight\":0.5},\"type\":\"insert\"}",
+			"{\"data\":{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.18},\"type\":\"delete\"}",
+			"{\"data\":{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.17},\"type\":\"insert\"}",
+			"{\"data\":{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.17},\"type\":\"delete\"}",
+			"{\"data\":{\"id\":101,\"name\":\"scooter\",\"description\":\"Small 2-wheel scooter\",\"weight\":3.14},\"type\":\"delete\"}",
+			"{\"data\":{\"id\":101,\"name\":\"scooter\",\"description\":\"Small 2-wheel scooter\",\"weight\":5.17},\"type\":\"insert\"}",
+			"{\"data\":{\"id\":102,\"name\":\"car battery\",\"description\":\"12V car battery\",\"weight\":8.1},\"type\":\"delete\"}",
+			"{\"data\":{\"id\":102,\"name\":\"car battery\",\"description\":\"12V car battery\",\"weight\":5.17},\"type\":\"insert\"}",
+			"{\"data\":{\"id\":102,\"name\":\"car battery\",\"description\":\"12V car battery\",\"weight\":5.17},\"type\":\"delete\"}",
+			"{\"data\":{\"id\":103,\"name\":\"12-pack drill bits\",\"description\":\"12-pack of drill bits with sizes ranging from #40 to #3\",\"weight\":0.8},\"type\":\"delete\"}"
+		);
+		assertEquals(expectedResult, result);
+	}
+
+	// --------------------------------------------------------------------------------------------
+	// Utilities
+	// --------------------------------------------------------------------------------------------
+
+	private static List<String> readLines(String resource) throws IOException {
+		final URL url = MaxwellJsonSerDerTest.class.getClassLoader().getResource(resource);
+		assert url != null;
+		Path path = new File(url.getFile()).toPath();
+		return Files.readAllLines(path);
+	}
+
+	private static class SimpleCollector implements Collector<RowData> {
+
+		private List<RowData> list = new ArrayList<>();
+
+		@Override
+		public void collect(RowData record) {
+			list.add(record);
+		}
+
+		@Override
+		public void close() {
+			// do nothing
+		}
+	}
+}
diff --git a/flink-formats/flink-json/src/test/resources/maxwell-data.txt b/flink-formats/flink-json/src/test/resources/maxwell-data.txt
new file mode 100644
index 0000000..ecba573
--- /dev/null
+++ b/flink-formats/flink-json/src/test/resources/maxwell-data.txt
@@ -0,0 +1,20 @@
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":0,"data":{"id":101,"name":"scooter","description":"Small 2-wheel scooter","weight":3.14}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":1,"data":{"id":102,"name":"car battery","description":"12V car battery","weight":8.1}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":2,"data":{"id":103,"name":"12-pack drill bits","description":"12-pack of drill bits with sizes ranging from #40 to #3","weight":0.8}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":3,"data":{"id":104,"name":"hammer","description":"12oz carpenter's hammer","weight":0.75}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":4,"data":{"id":105,"name":"hammer","description":"14oz carpenter's hammer","weight":0.875}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":5,"data":{"id":106,"name":"hammer","description":"16oz carpenter's hammer","weight":1.0}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":6,"data":{"id":107,"name":"rocks","description":"box of assorted rocks","weight":5.3}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"xoffset":7,"data":{"id":108,"name":"jacket","description":"water resistent black wind breaker","weight":0.1}}
+{"database":"test","table":"product","type":"insert","ts":1596684883,"xid":7125,"commit":true,"data":{"id":109,"name":"spare tire","description":"24 inch spare tire","weight":22.2}}
+{"database":"test","table":"product","type":"update","ts":1596684893,"xid":7152,"commit":true,"data":{"id":106,"name":"hammer","description":"18oz carpenter hammer","weight":1.0},"old":{"description":"16oz carpenter's hammer"}}
+{"database":"test","table":"product","type":"update","ts":1596684897,"xid":7169,"commit":true,"data":{"id":107,"name":"rocks","description":"box of assorted rocks","weight":5.1},"old":{"weight":5.3}}
+{"database":"test","table":"product","type":"insert","ts":1596684900,"xid":7186,"commit":true,"data":{"id":110,"name":"jacket","description":"water resistent white wind breaker","weight":0.2}}
+{"database":"test","table":"product","type":"insert","ts":1596684904,"xid":7201,"commit":true,"data":{"id":111,"name":"scooter","description":"Big 2-wheel scooter ","weight":5.18}}
+{"database":"test","table":"product","type":"update","ts":1596684906,"xid":7216,"commit":true,"data":{"id":110,"name":"jacket","description":"new water resistent white wind breaker","weight":0.5},"old":{"description":"water resistent white wind breaker","weight":0.2}}
+{"database":"test","table":"product","type":"update","ts":1596684912,"xid":7235,"commit":true,"data":{"id":111,"name":"scooter","description":"Big 2-wheel scooter ","weight":5.17},"old":{"weight":5.18}}
+{"database":"test","table":"product","type":"delete","ts":1596684914,"xid":7250,"commit":true,"data":{"id":111,"name":"scooter","description":"Big 2-wheel scooter ","weight":5.17}}
+{"database":"test","table":"product","type":"update","ts":1596684928,"xid":7291,"xoffset":0,"data":{"id":101,"name":"scooter","description":"Small 2-wheel scooter","weight":5.17},"old":{"weight":3.14}}
+{"database":"test","table":"product","type":"update","ts":1596684928,"xid":7291,"commit":true,"data":{"id":102,"name":"car battery","description":"12V car battery","weight":5.17},"old":{"weight":8.1}}
+{"database":"test","table":"product","type":"delete","ts":1596684938,"xid":7322,"xoffset":0,"data":{"id":102,"name":"car battery","description":"12V car battery","weight":5.17}}
+{"database":"test","table":"product","type":"delete","ts":1596684938,"xid":7322,"commit":true,"data":{"id":103,"name":"12-pack drill bits","description":"12-pack of drill bits with sizes ranging from #40 to #3","weight":0.8}}