You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by di...@apache.org on 2024/09/25 02:55:03 UTC
(flink) 02/03: [FLINK-36311][format/json] Remove deprecated APIs in Flink Json format
This is an automated email from the ASF dual-hosted git repository.
dianfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 98a0a567edf6bf660d9629c0e5e12e81b443a820
Author: Dian Fu <di...@apache.org>
AuthorDate: Wed Sep 18 15:00:14 2024 +0800
[FLINK-36311][format/json] Remove deprecated APIs in Flink Json format
---
.../json/JsonNodeDeserializationSchema.java | 37 --
.../json/JsonNodeDeserializationSchemaTest.java | 50 ---
.../json/JsonRowDeserializationSchemaTest.java | 443 ---------------------
.../json/JsonRowSerializationSchemaTest.java | 250 ------------
flink-python/pom.xml | 14 +-
.../formats/json/JsonRowDeserializationSchema.java | 25 +-
.../formats/json/JsonRowSerializationSchema.java | 14 +-
7 files changed, 13 insertions(+), 820 deletions(-)
diff --git a/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/JsonNodeDeserializationSchema.java b/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/JsonNodeDeserializationSchema.java
deleted file mode 100644
index 928a6f1e59d..00000000000
--- a/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/JsonNodeDeserializationSchema.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.formats.json;
-
-import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode;
-
-/**
- * DeserializationSchema that deserializes a JSON String into an ObjectNode.
- *
- * <p>Fields can be accessed by calling objectNode.get(<name>).as(<type>)
- *
- * @deprecated Use {@code new JsonDeserializationSchema(ObjectNode.class)} instead
- */
-@Deprecated
-public class JsonNodeDeserializationSchema extends JsonDeserializationSchema<ObjectNode> {
-
- private static final long serialVersionUID = 2L;
-
- public JsonNodeDeserializationSchema() {
- super(ObjectNode.class);
- }
-}
diff --git a/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/JsonNodeDeserializationSchemaTest.java b/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/JsonNodeDeserializationSchemaTest.java
deleted file mode 100644
index e6b2a3e05cb..00000000000
--- a/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/JsonNodeDeserializationSchemaTest.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.formats.json;
-
-import org.apache.flink.connector.testutils.formats.DummyInitializationContext;
-import org.apache.flink.util.jackson.JacksonMapperFactory;
-
-import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode;
-
-import org.junit.jupiter.api.Test;
-
-import java.io.IOException;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-/** Tests for the {@link JsonNodeDeserializationSchema}. */
-@SuppressWarnings("deprecation")
-class JsonNodeDeserializationSchemaTest {
-
- @Test
- void testDeserialize() throws IOException {
- ObjectMapper mapper = JacksonMapperFactory.createObjectMapper();
- ObjectNode initialValue = mapper.createObjectNode();
- initialValue.put("key", 4).put("value", "world");
- byte[] serializedValue = mapper.writeValueAsBytes(initialValue);
-
- JsonNodeDeserializationSchema schema = new JsonNodeDeserializationSchema();
- schema.open(new DummyInitializationContext());
- ObjectNode deserializedValue = schema.deserialize(serializedValue);
-
- assertThat(deserializedValue.get("key").asInt()).isEqualTo(4);
- assertThat(deserializedValue.get("value").asText()).isEqualTo("world");
- }
-}
diff --git a/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/JsonRowDeserializationSchemaTest.java b/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/JsonRowDeserializationSchemaTest.java
deleted file mode 100644
index 4712768f291..00000000000
--- a/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/JsonRowDeserializationSchemaTest.java
+++ /dev/null
@@ -1,443 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.formats.json;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeinfo.Types;
-import org.apache.flink.types.Row;
-import org.apache.flink.util.jackson.JacksonMapperFactory;
-
-import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode;
-
-import org.junit.jupiter.api.Test;
-
-import javax.annotation.Nullable;
-
-import java.math.BigDecimal;
-import java.sql.Date;
-import java.sql.Time;
-import java.sql.Timestamp;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ThreadLocalRandom;
-
-import static org.apache.flink.formats.utils.DeserializationSchemaMatcher.whenDeserializedWith;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-
-/** Tests for the {@link JsonRowDeserializationSchema}. */
-public class JsonRowDeserializationSchemaTest {
-
- private static final ObjectMapper OBJECT_MAPPER = JacksonMapperFactory.createObjectMapper();
-
- /** Tests simple deserialization using type information. */
- @Test
- public void testTypeInfoDeserialization() throws Exception {
- long id = 1238123899121L;
- String name = "asdlkjasjkdla998y1122";
- byte[] bytes = new byte[1024];
- ThreadLocalRandom.current().nextBytes(bytes);
- Timestamp timestamp = Timestamp.valueOf("1990-10-14 12:12:43");
- Date date = Date.valueOf("1990-10-14");
- Time time = Time.valueOf("12:12:43");
-
- Map<String, Long> map = new HashMap<>();
- map.put("flink", 123L);
-
- Map<String, Map<String, Integer>> nestedMap = new HashMap<>();
- Map<String, Integer> innerMap = new HashMap<>();
- innerMap.put("key", 234);
- nestedMap.put("inner_map", innerMap);
-
- // Root
- ObjectNode root = OBJECT_MAPPER.createObjectNode();
- root.put("id", id);
- root.put("name", name);
- root.put("bytes", bytes);
- root.put("date1", "1990-10-14");
- root.put("date2", "1990-10-14");
- root.put("time1", "12:12:43Z");
- root.put("time2", "12:12:43Z");
- root.put("timestamp1", "1990-10-14T12:12:43Z");
- root.put("timestamp2", "1990-10-14T12:12:43Z");
- root.putObject("map").put("flink", 123);
- root.putObject("map2map").putObject("inner_map").put("key", 234);
-
- byte[] serializedJson = OBJECT_MAPPER.writeValueAsBytes(root);
-
- JsonRowDeserializationSchema deserializationSchema =
- new JsonRowDeserializationSchema.Builder(
- Types.ROW_NAMED(
- new String[] {
- "id",
- "name",
- "bytes",
- "date1",
- "date2",
- "time1",
- "time2",
- "timestamp1",
- "timestamp2",
- "map",
- "map2map"
- },
- Types.LONG,
- Types.STRING,
- Types.PRIMITIVE_ARRAY(Types.BYTE),
- Types.SQL_DATE,
- Types.LOCAL_DATE,
- Types.SQL_TIME,
- Types.LOCAL_TIME,
- Types.SQL_TIMESTAMP,
- Types.LOCAL_DATE_TIME,
- Types.MAP(Types.STRING, Types.LONG),
- Types.MAP(
- Types.STRING, Types.MAP(Types.STRING, Types.INT))))
- .build();
-
- Row row = new Row(11);
- row.setField(0, id);
- row.setField(1, name);
- row.setField(2, bytes);
- row.setField(3, date);
- row.setField(4, date.toLocalDate());
- row.setField(5, time);
- row.setField(6, time.toLocalTime());
- row.setField(7, timestamp);
- row.setField(8, timestamp.toLocalDateTime());
- row.setField(9, map);
- row.setField(10, nestedMap);
-
- assertThat(
- whenDeserializedWith(deserializationSchema)
- .equalsTo(row)
- .matches(serializedJson))
- .isTrue();
- }
-
- @Test
- public void testSchemaDeserialization() throws Exception {
- final BigDecimal id = BigDecimal.valueOf(1238123899121L);
- final String name = "asdlkjasjkdla998y1122";
- final byte[] bytes = new byte[1024];
- ThreadLocalRandom.current().nextBytes(bytes);
- final BigDecimal[] numbers =
- new BigDecimal[] {
- BigDecimal.valueOf(1), BigDecimal.valueOf(2), BigDecimal.valueOf(3)
- };
- final String[] strings = new String[] {"one", "two", "three"};
-
- // Root
- ObjectNode root = OBJECT_MAPPER.createObjectNode();
- root.put("id", id.longValue());
- root.putNull("idOrNull");
- root.put("name", name);
- root.put("date", "1990-10-14");
- root.put("time", "12:12:43Z");
- root.put("timestamp", "1990-10-14T12:12:43Z");
- root.put("bytes", bytes);
- root.putArray("numbers").add(1).add(2).add(3);
- root.putArray("strings").add("one").add("two").add("three");
- root.putObject("nested").put("booleanField", true).put("decimalField", 12);
-
- final byte[] serializedJson = OBJECT_MAPPER.writeValueAsBytes(root);
-
- JsonRowDeserializationSchema deserializationSchema =
- new JsonRowDeserializationSchema.Builder(
- "{"
- + " type: 'object',"
- + " properties: {"
- + " id: { type: 'integer' },"
- + " idOrNull: { type: ['integer', 'null'] },"
- + " name: { type: 'string' },"
- + " date: { type: 'string', format: 'date' },"
- + " time: { type: 'string', format: 'time' },"
- + " timestamp: { type: 'string', format: 'date-time' },"
- + " bytes: { type: 'string', contentEncoding: 'base64' },"
- + " numbers: { type: 'array', items: { type: 'integer' } },"
- + " strings: { type: 'array', items: { type: 'string' } },"
- + " nested: { "
- + " type: 'object',"
- + " properties: { "
- + " booleanField: { type: 'boolean' },"
- + " decimalField: { type: 'number' }"
- + " }"
- + " }"
- + " }"
- + "}")
- .build();
-
- final Row expected = new Row(10);
- expected.setField(0, id);
- expected.setField(1, null);
- expected.setField(2, name);
- expected.setField(3, Date.valueOf("1990-10-14"));
- expected.setField(4, Time.valueOf("12:12:43"));
- expected.setField(5, Timestamp.valueOf("1990-10-14 12:12:43"));
- expected.setField(6, bytes);
- expected.setField(7, numbers);
- expected.setField(8, strings);
- final Row nestedRow = new Row(2);
- nestedRow.setField(0, true);
- nestedRow.setField(1, BigDecimal.valueOf(12));
- expected.setField(9, nestedRow);
-
- assertThat(
- whenDeserializedWith(deserializationSchema)
- .equalsTo(expected)
- .matches(serializedJson))
- .isTrue();
- }
-
- /** Tests deserialization with non-existing field name. */
- @Test
- public void testMissingNode() throws Exception {
- // Root
- ObjectNode root = OBJECT_MAPPER.createObjectNode();
- root.put("id", 123123123);
- byte[] serializedJson = OBJECT_MAPPER.writeValueAsBytes(root);
-
- TypeInformation<Row> rowTypeInformation =
- Types.ROW_NAMED(new String[] {"name"}, Types.STRING);
-
- JsonRowDeserializationSchema deserializationSchema =
- new JsonRowDeserializationSchema.Builder(rowTypeInformation).build();
-
- Row row = new Row(1);
- assertThat(
- whenDeserializedWith(deserializationSchema)
- .equalsTo(row)
- .matches(serializedJson))
- .isTrue();
-
- deserializationSchema =
- new JsonRowDeserializationSchema.Builder(rowTypeInformation)
- .failOnMissingField()
- .build();
- final JsonRowDeserializationSchema errorDs = deserializationSchema;
- assertThatThrownBy(() -> errorDs.deserialize(serializedJson))
- .isInstanceOf(Exception.class)
- .hasMessageContaining("Failed to deserialize JSON");
-
- // ignore-parse-errors ignores missing field exception too
- deserializationSchema =
- new JsonRowDeserializationSchema.Builder(rowTypeInformation)
- .ignoreParseErrors()
- .build();
- assertThat(
- whenDeserializedWith(deserializationSchema)
- .equalsTo(row)
- .matches(serializedJson))
- .isTrue();
-
- assertThatThrownBy(
- () ->
- new JsonRowDeserializationSchema.Builder(rowTypeInformation)
- .failOnMissingField()
- .ignoreParseErrors()
- .build())
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining(
- "JSON format doesn't support failOnMissingField and ignoreParseErrors are both true");
- }
-
- /** Tests that number of field names and types has to match. */
- @Test
- public void testNumberOfFieldNamesAndTypesMismatch() {
- assertThatThrownBy(
- () ->
- new JsonRowDeserializationSchema.Builder(
- Types.ROW_NAMED(
- new String[] {"one", "two", "three"},
- Types.LONG))
- .build())
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining("Number of field types and names is different.");
- }
-
- @Test
- public void testJsonParse() {
- for (TestSpec spec : testData) {
- testIgnoreParseErrors(spec);
- if (spec.errorMessage != null) {
- testParseErrors(spec);
- }
- }
- }
-
- private void testIgnoreParseErrors(TestSpec spec) {
- // the parsing field should be null and no exception is thrown
- JsonRowDeserializationSchema ignoreErrorsSchema =
- new JsonRowDeserializationSchema.Builder(spec.rowTypeInformation)
- .ignoreParseErrors()
- .build();
- Row expected;
- if (spec.expected != null) {
- expected = spec.expected;
- } else {
- expected = new Row(1);
- }
-
- assertThat(
- whenDeserializedWith(ignoreErrorsSchema)
- .equalsTo(expected)
- .matches(spec.json.getBytes()))
- .isTrue();
- }
-
- private void testParseErrors(TestSpec spec) {
- // expect exception if parse error is not ignored
- JsonRowDeserializationSchema failingSchema =
- new JsonRowDeserializationSchema.Builder(spec.rowTypeInformation).build();
-
- assertThatThrownBy(() -> failingSchema.deserialize(spec.json.getBytes()))
- .isInstanceOf(Exception.class)
- .hasMessageContaining(spec.errorMessage);
- }
-
- private static List<TestSpec> testData =
- Arrays.asList(
- TestSpec.json("{\"id\": \"trueA\"}")
- .typeInfo(Types.ROW_NAMED(new String[] {"id"}, Types.BOOLEAN))
- .expect(Row.of(false)),
- TestSpec.json("{\"id\": true}")
- .typeInfo(Types.ROW_NAMED(new String[] {"id"}, Types.BOOLEAN))
- .expect(Row.of(true)),
- TestSpec.json("{\"id\":\"abc\"}")
- .typeInfo(Types.ROW_NAMED(new String[] {"id"}, Types.INT))
- .expectErrorMessage("Failed to deserialize JSON '{\"id\":\"abc\"}'"),
- TestSpec.json("{\"id\":112.013}")
- .typeInfo(Types.ROW_NAMED(new String[] {"id"}, Types.LONG))
- .expect(Row.of(112L)),
- TestSpec.json("{\"id\":true}")
- .typeInfo(Types.ROW_NAMED(new String[] {"id"}, Types.STRING))
- .expect(Row.of("true")),
- TestSpec.json("{\"id\":123.234}")
- .typeInfo(Types.ROW_NAMED(new String[] {"id"}, Types.STRING))
- .expect(Row.of("123.234")),
- TestSpec.json("{\"id\":1234567}")
- .typeInfo(Types.ROW_NAMED(new String[] {"id"}, Types.STRING))
- .expect(Row.of("1234567")),
- TestSpec.json("{\"id\":\"string field\"}")
- .typeInfo(Types.ROW_NAMED(new String[] {"id"}, Types.STRING))
- .expect(Row.of("string field")),
- TestSpec.json("{\"id\":[\"array data1\",\"array data2\",123,234.345]}")
- .typeInfo(Types.ROW_NAMED(new String[] {"id"}, Types.STRING))
- .expect(Row.of("[\"array data1\",\"array data2\",123,234.345]")),
- TestSpec.json("{\"id\":{\"k1\":123,\"k2\":234.234,\"k3\":\"string data\"}}")
- .typeInfo(Types.ROW_NAMED(new String[] {"id"}, Types.STRING))
- .expect(Row.of("{\"k1\":123,\"k2\":234.234,\"k3\":\"string data\"}")),
- TestSpec.json("{\"id\":\"long\"}")
- .typeInfo(Types.ROW_NAMED(new String[] {"id"}, Types.LONG))
- .expectErrorMessage("Failed to deserialize JSON '{\"id\":\"long\"}'"),
- TestSpec.json("{\"id\":\"112.013.123\"}")
- .typeInfo(Types.ROW_NAMED(new String[] {"id"}, Types.FLOAT))
- .expectErrorMessage(
- "Failed to deserialize JSON '{\"id\":\"112.013.123\"}'"),
- TestSpec.json("{\"id\":\"112.013.123\"}")
- .typeInfo(Types.ROW_NAMED(new String[] {"id"}, Types.DOUBLE))
- .expectErrorMessage(
- "Failed to deserialize JSON '{\"id\":\"112.013.123\"}'"),
- TestSpec.json("{\"id\":\"18:00:243\"}")
- .typeInfo(Types.ROW_NAMED(new String[] {"id"}, Types.SQL_TIME))
- .expectErrorMessage(
- "Failed to deserialize JSON '{\"id\":\"18:00:243\"}'"),
- TestSpec.json("{\"id\":\"20191112\"}")
- .typeInfo(Types.ROW_NAMED(new String[] {"id"}, Types.SQL_DATE))
- .expectErrorMessage(
- "Failed to deserialize JSON '{\"id\":\"20191112\"}'"),
- TestSpec.json("{\"id\":\"2019-11-12 18:00:12\"}")
- .typeInfo(Types.ROW_NAMED(new String[] {"id"}, Types.SQL_TIMESTAMP))
- .expectErrorMessage(
- "Failed to deserialize JSON '{\"id\":\"2019-11-12 18:00:12\"}'"),
- TestSpec.json("{\"id\":\"abc\"}")
- .typeInfo(Types.ROW_NAMED(new String[] {"id"}, Types.BIG_DEC))
- .expectErrorMessage("Failed to deserialize JSON '{\"id\":\"abc\"}'"),
- TestSpec.json("{\"row\":{\"id\":\"abc\"}}")
- .typeInfo(
- Types.ROW_NAMED(
- new String[] {"row"},
- Types.ROW_NAMED(new String[] {"id"}, Types.INT)))
- .expect(Row.of(new Row(1)))
- .expectErrorMessage(
- "Failed to deserialize JSON '{\"row\":{\"id\":\"abc\"}}'"),
- TestSpec.json("{\"array\":[123, \"abc\"]}")
- .typeInfo(
- Types.ROW_NAMED(
- new String[] {"array"}, Types.OBJECT_ARRAY(Types.INT)))
- .expect(Row.of((Object) new Integer[] {123, null}))
- .expectErrorMessage(
- "Failed to deserialize JSON '{\"array\":[123, \"abc\"]}'"),
- TestSpec.json("{\"map\":{\"key1\":\"123\", \"key2\":\"abc\"}}")
- .typeInfo(
- Types.ROW_NAMED(
- new String[] {"map"},
- Types.MAP(Types.STRING, Types.INT)))
- .expect(Row.of(createHashMap("key1", 123, "key2", null)))
- .expectErrorMessage(
- "Failed to deserialize JSON '{\"map\":{\"key1\":\"123\", \"key2\":\"abc\"}}'"),
- TestSpec.json("{\"id\":1,\"factor\":799.929496989092949698}")
- .typeInfo(
- Types.ROW_NAMED(
- new String[] {"id", "factor"},
- Types.INT,
- Types.BIG_DEC))
- .expect(Row.of(1, new BigDecimal("799.929496989092949698"))));
-
- private static Map<String, Integer> createHashMap(
- String k1, Integer v1, String k2, Integer v2) {
- Map<String, Integer> map = new HashMap<>();
- map.put(k1, v1);
- map.put(k2, v2);
- return map;
- }
-
- private static class TestSpec {
- private final String json;
- private @Nullable TypeInformation<Row> rowTypeInformation;
- private @Nullable Row expected;
- private @Nullable String errorMessage;
-
- private TestSpec(String json) {
- this.json = json;
- }
-
- public static TestSpec json(String json) {
- return new TestSpec(json);
- }
-
- TestSpec expect(Row row) {
- this.expected = row;
- return this;
- }
-
- TestSpec typeInfo(TypeInformation<Row> rowTypeInformation) {
- this.rowTypeInformation = rowTypeInformation;
- return this;
- }
-
- TestSpec expectErrorMessage(String errorMessage) {
- this.errorMessage = errorMessage;
- return this;
- }
- }
-}
diff --git a/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/JsonRowSerializationSchemaTest.java b/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/JsonRowSerializationSchemaTest.java
deleted file mode 100644
index 5140a8556a4..00000000000
--- a/flink-formats/flink-json/src/test/java/org/apache/flink/formats/json/JsonRowSerializationSchemaTest.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.formats.json;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeinfo.Types;
-import org.apache.flink.types.Row;
-
-import org.junit.jupiter.api.Test;
-
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.sql.Date;
-import java.sql.Time;
-import java.sql.Timestamp;
-import java.util.concurrent.ThreadLocalRandom;
-
-import static org.apache.flink.connector.testutils.formats.SchemaTestUtils.open;
-import static org.apache.flink.formats.utils.SerializationSchemaMatcher.whenSerializedWith;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-
-/** Tests for the {@link JsonRowSerializationSchema}. */
-public class JsonRowSerializationSchemaTest {
-
- @Test
- public void testRowSerialization() {
- final TypeInformation<Row> rowSchema =
- Types.ROW_NAMED(
- new String[] {"f1", "f2", "f3", "f4", "f5"},
- Types.INT,
- Types.BOOLEAN,
- Types.STRING,
- Types.SQL_TIMESTAMP,
- Types.LOCAL_DATE_TIME);
-
- final Row row = new Row(5);
- row.setField(0, 1);
- row.setField(1, true);
- row.setField(2, "str");
- row.setField(3, Timestamp.valueOf("1990-10-14 12:12:43"));
- row.setField(4, Timestamp.valueOf("1990-10-14 12:12:43").toLocalDateTime());
-
- final JsonRowSerializationSchema serializationSchema =
- new JsonRowSerializationSchema.Builder(rowSchema).build();
- final JsonRowDeserializationSchema deserializationSchema =
- new JsonRowDeserializationSchema.Builder(rowSchema).build();
-
- assertThat(
- whenSerializedWith(serializationSchema)
- .andDeserializedWith(deserializationSchema)
- .equalsTo(row)
- .matches(row))
- .isTrue();
- }
-
- @Test
- public void testSerializationOfTwoRows() throws IOException {
- final TypeInformation<Row> rowSchema =
- Types.ROW_NAMED(
- new String[] {"f1", "f2", "f3"}, Types.INT, Types.BOOLEAN, Types.STRING);
-
- final Row row1 = new Row(3);
- row1.setField(0, 1);
- row1.setField(1, true);
- row1.setField(2, "str");
-
- final JsonRowSerializationSchema serializationSchema =
- new JsonRowSerializationSchema.Builder(rowSchema).build();
- open(serializationSchema);
- final JsonRowDeserializationSchema deserializationSchema =
- new JsonRowDeserializationSchema.Builder(rowSchema).build();
- open(deserializationSchema);
-
- byte[] bytes = serializationSchema.serialize(row1);
- assertThat(deserializationSchema.deserialize(bytes)).isEqualTo(row1);
-
- final Row row2 = new Row(3);
- row2.setField(0, 10);
- row2.setField(1, false);
- row2.setField(2, "newStr");
-
- bytes = serializationSchema.serialize(row2);
- assertThat(deserializationSchema.deserialize(bytes)).isEqualTo(row2);
- }
-
- @Test
- public void testMultiRowsWithNullValues() throws IOException {
- String[] jsons =
- new String[] {
- "{\"svt\":\"2020-02-24T12:58:09.209+0800\"}",
- "{\"svt\":\"2020-02-24T12:58:09.209+0800\", \"ops\":{\"id\":\"281708d0-4092-4c21-9233-931950b6eccf\"}, "
- + "\"ids\":[1, 2, 3]}",
- "{\"svt\":\"2020-02-24T12:58:09.209+0800\"}",
- };
-
- String[] expected =
- new String[] {
- "{\"svt\":\"2020-02-24T12:58:09.209+0800\",\"ops\":null,\"ids\":null}",
- "{\"svt\":\"2020-02-24T12:58:09.209+0800\",\"ops\":{\"id\":\"281708d0-4092-4c21-9233-931950b6eccf\"},"
- + "\"ids\":[1,2,3]}",
- "{\"svt\":\"2020-02-24T12:58:09.209+0800\",\"ops\":null,\"ids\":null}",
- };
-
- TypeInformation<Row> schema =
- Types.ROW_NAMED(
- new String[] {"svt", "ops", "ids"},
- Types.STRING,
- Types.ROW_NAMED(new String[] {"id"}, Types.STRING),
- Types.PRIMITIVE_ARRAY(Types.INT));
- JsonRowDeserializationSchema deserializationSchema =
- new JsonRowDeserializationSchema.Builder(schema).build();
- open(deserializationSchema);
- JsonRowSerializationSchema serializationSchema =
- JsonRowSerializationSchema.builder().withTypeInfo(schema).build();
- open(serializationSchema);
-
- for (int i = 0; i < jsons.length; i++) {
- String json = jsons[i];
- Row row = deserializationSchema.deserialize(json.getBytes());
- String result = new String(serializationSchema.serialize(row));
- assertThat(result).isEqualTo(expected[i]);
- }
- }
-
- @Test
- public void testNestedSchema() {
- final TypeInformation<Row> rowSchema =
- Types.ROW_NAMED(
- new String[] {"f1", "f2", "f3"},
- Types.INT,
- Types.BOOLEAN,
- Types.ROW(Types.INT, Types.DOUBLE));
-
- final Row row = new Row(3);
- row.setField(0, 42);
- row.setField(1, false);
- final Row nested = new Row(2);
- nested.setField(0, 22);
- nested.setField(1, 2.3);
- row.setField(2, nested);
-
- final JsonRowSerializationSchema serializationSchema =
- new JsonRowSerializationSchema.Builder(rowSchema).build();
- final JsonRowDeserializationSchema deserializationSchema =
- new JsonRowDeserializationSchema.Builder(rowSchema).build();
-
- assertThat(
- whenSerializedWith(serializationSchema)
- .andDeserializedWith(deserializationSchema)
- .equalsTo(row)
- .matches(row))
- .isTrue();
- }
-
- @Test
- public void testSerializeRowWithInvalidNumberOfFields() {
- final TypeInformation<Row> rowSchema =
- Types.ROW_NAMED(
- new String[] {"f1", "f2", "f3"}, Types.INT, Types.BOOLEAN, Types.STRING);
-
- final Row row = new Row(1);
- row.setField(0, 1);
-
- final JsonRowSerializationSchema serializationSchema =
- new JsonRowSerializationSchema.Builder(rowSchema).build();
- open(serializationSchema);
- assertThatThrownBy(() -> serializationSchema.serialize(row))
- .isInstanceOf(RuntimeException.class);
- }
-
- @Test
- public void testSchema() {
- final TypeInformation<Row> rowSchema =
- JsonRowSchemaConverter.convert(
- "{"
- + " type: 'object',"
- + " properties: {"
- + " id: { type: 'integer' },"
- + " idNumber: { type: 'number' },"
- + " idOrNull: { type: ['integer', 'null'] },"
- + " name: { type: 'string' },"
- + " date: { type: 'string', format: 'date' },"
- + " time: { type: 'string', format: 'time' },"
- + " timestamp: { type: 'string', format: 'date-time' },"
- + " bytes: { type: 'string', contentEncoding: 'base64' },"
- + " numbers: { type: 'array', items: { type: 'integer' } },"
- + " strings: { type: 'array', items: { type: 'string' } },"
- + " nested: { "
- + " type: 'object',"
- + " properties: { "
- + " booleanField: { type: 'boolean' },"
- + " decimalField: { type: 'number' }"
- + " }"
- + " }"
- + " }"
- + "}");
-
- final Row row = new Row(11);
- row.setField(0, BigDecimal.valueOf(-333));
- row.setField(1, BigDecimal.valueOf(12.2222));
- row.setField(2, null);
- row.setField(3, "");
- row.setField(4, Date.valueOf("1990-10-14"));
- row.setField(5, Time.valueOf("12:12:43"));
- row.setField(6, Timestamp.valueOf("1990-10-14 12:12:43"));
-
- final byte[] bytes = new byte[1024];
- ThreadLocalRandom.current().nextBytes(bytes);
- row.setField(7, bytes);
- final BigDecimal[] numbers =
- new BigDecimal[] {
- BigDecimal.valueOf(1), BigDecimal.valueOf(2), BigDecimal.valueOf(3)
- };
- row.setField(8, numbers);
- final String[] strings = new String[] {"one", "two", "three"};
- row.setField(9, strings);
- final Row nestedRow = new Row(2);
- nestedRow.setField(0, true);
- nestedRow.setField(1, BigDecimal.valueOf(12));
- row.setField(10, nestedRow);
-
- final JsonRowSerializationSchema serializationSchema =
- new JsonRowSerializationSchema.Builder(rowSchema).build();
- final JsonRowDeserializationSchema deserializationSchema =
- new JsonRowDeserializationSchema.Builder(rowSchema).build();
-
- assertThat(
- whenSerializedWith(serializationSchema)
- .andDeserializedWith(deserializationSchema)
- .equalsTo(row)
- .matches(row))
- .isTrue();
- }
-}
diff --git a/flink-python/pom.xml b/flink-python/pom.xml
index 697a6d53cc3..1f96abd752d 100644
--- a/flink-python/pom.xml
+++ b/flink-python/pom.xml
@@ -103,6 +103,12 @@ under the License.
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-json</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
<!-- Beam dependencies -->
@@ -284,14 +290,6 @@ under the License.
<scope>test</scope>
</dependency>
- <dependency>
- <!-- Indirectly accessed in pyflink_gateway_server -->
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-json</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
<dependency>
<!-- Indirectly accessed in pyflink_gateway_server -->
<groupId>org.apache.flink</groupId>
diff --git a/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/JsonRowDeserializationSchema.java b/flink-python/src/main/java/org/apache/flink/formats/json/JsonRowDeserializationSchema.java
similarity index 96%
rename from flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/JsonRowDeserializationSchema.java
rename to flink-python/src/main/java/org/apache/flink/formats/json/JsonRowDeserializationSchema.java
index dd4a9bb9f99..ec00116ebd3 100644
--- a/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/JsonRowDeserializationSchema.java
+++ b/flink-python/src/main/java/org/apache/flink/formats/json/JsonRowDeserializationSchema.java
@@ -18,7 +18,7 @@
package org.apache.flink.formats.json;
-import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.serialization.DeserializationSchema;
import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
@@ -78,13 +78,8 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
* <p>Deserializes a <code>byte[]</code> message as a JSON object and reads the specified fields.
*
* <p>Failures during deserialization are forwarded as wrapped IOExceptions.
- *
- * @deprecated The format was developed for the Table API users and will not be maintained for
- * DataStream API users anymore. Either use Table API or switch to Data Stream, defining your
- * own {@link DeserializationSchema}.
*/
-@PublicEvolving
-@Deprecated
+@Internal
public class JsonRowDeserializationSchema implements DeserializationSchema<Row> {
private static final long serialVersionUID = -228294330688809195L;
@@ -92,14 +87,14 @@ public class JsonRowDeserializationSchema implements DeserializationSchema<Row>
/** Type information describing the result type. */
private final RowTypeInfo typeInfo;
- private boolean failOnMissingField;
+ private final boolean failOnMissingField;
private final boolean hasDecimalType;
/** Object mapper for parsing the JSON. */
private transient ObjectMapper objectMapper;
- private DeserializationRuntimeConverter runtimeConverter;
+ private final DeserializationRuntimeConverter runtimeConverter;
/** Flag indicating whether to ignore invalid fields/rows (default: throw an exception). */
private final boolean ignoreParseErrors;
@@ -128,26 +123,14 @@ public class JsonRowDeserializationSchema implements DeserializationSchema<Row>
}
}
- /** @deprecated Use the provided {@link Builder} instead. */
- @Deprecated
public JsonRowDeserializationSchema(TypeInformation<Row> typeInfo) {
this(typeInfo, false, false);
}
- /** @deprecated Use the provided {@link Builder} instead. */
- @Deprecated
public JsonRowDeserializationSchema(String jsonSchema) {
this(JsonRowSchemaConverter.convert(checkNotNull(jsonSchema)), false, false);
}
- /** @deprecated Use the provided {@link Builder} instead. */
- @Deprecated
- public void setFailOnMissingField(boolean failOnMissingField) {
- // TODO make this class immutable once we drop this method
- this.failOnMissingField = failOnMissingField;
- this.runtimeConverter = createConverter(this.typeInfo);
- }
-
@Override
public Row deserialize(byte[] message) throws IOException {
try {
diff --git a/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/JsonRowSerializationSchema.java b/flink-python/src/main/java/org/apache/flink/formats/json/JsonRowSerializationSchema.java
similarity index 96%
rename from flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/JsonRowSerializationSchema.java
rename to flink-python/src/main/java/org/apache/flink/formats/json/JsonRowSerializationSchema.java
index f185d211bf9..90374ee5edc 100644
--- a/flink-formats/flink-json/src/main/java/org/apache/flink/formats/json/JsonRowSerializationSchema.java
+++ b/flink-python/src/main/java/org/apache/flink/formats/json/JsonRowSerializationSchema.java
@@ -17,6 +17,7 @@
package org.apache.flink.formats.json;
+import org.apache.flink.annotation.Internal;
import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.api.common.serialization.SerializationSchema;
import org.apache.flink.api.common.typeinfo.BasicArrayTypeInfo;
@@ -65,13 +66,8 @@ import static org.apache.flink.util.Preconditions.checkNotNull;
*
* <p>Result <code>byte[]</code> messages can be deserialized using {@link
* JsonRowDeserializationSchema}.
- *
- * @deprecated The format was developed for the Table API users and will not be maintained for
- * DataStream API users anymore. Either use Table API or switch to Data Stream, defining your
- * own {@link SerializationSchema}.
*/
-@PublicEvolving
-@Deprecated
+@Internal
public class JsonRowSerializationSchema implements SerializationSchema<Row> {
private static final long serialVersionUID = -2885556750743978636L;
@@ -115,9 +111,7 @@ public class JsonRowSerializationSchema implements SerializationSchema<Row> {
*
* @param typeInfo Type information describing the result type. The field names of {@link
* Row} are used to parse the JSON properties.
- * @deprecated Use {@link JsonRowSerializationSchema#builder()} instead.
*/
- @Deprecated
public Builder(TypeInformation<Row> typeInfo) {
checkArgument(typeInfo instanceof RowTypeInfo, "Only RowTypeInfo is supported");
this.typeInfo = (RowTypeInfo) typeInfo;
@@ -128,9 +122,7 @@ public class JsonRowSerializationSchema implements SerializationSchema<Row> {
*
* @param jsonSchema JSON schema describing the result type
* @see <a href="http://json-schema.org/">http://json-schema.org/</a>
- * @deprecated Use {@link JsonRowSerializationSchema#builder()} instead.
*/
- @Deprecated
public Builder(String jsonSchema) {
this(JsonRowSchemaConverter.convert(checkNotNull(jsonSchema)));
}
@@ -158,7 +150,7 @@ public class JsonRowSerializationSchema implements SerializationSchema<Row> {
}
}
- /** Creates a builder for {@link JsonRowSerializationSchema.Builder}. */
+ /** Creates a builder for {@link Builder}. */
public static Builder builder() {
return new Builder();
}