You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ga...@apache.org on 2023/03/17 06:44:03 UTC
[incubator-seatunnel] branch dev updated: [Feature][API] Support convert strings as List option (#4362)
This is an automated email from the ASF dual-hosted git repository.
gaojun2048 pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 4ad483357 [Feature][API] Support convert strings as List<T> option (#4362)
4ad483357 is described below
commit 4ad483357ab6a919cbfdc13866fadc0c07fa2a2a
Author: Zongwen Li <zo...@apache.org>
AuthorDate: Fri Mar 17 14:43:56 2023 +0800
[Feature][API] Support convert strings as List<T> option (#4362)
* [feature] `Option` supports splitting String into List<T>
---
.../api/configuration/ReadonlyConfig.java | 2 +-
.../api/configuration/util/ConfigUtil.java | 101 ++++++++++++++++++++-
.../api/configuration/ReadableConfigTest.java | 5 +
.../src/test/resources/conf/option-test.conf | 1 +
4 files changed, 107 insertions(+), 2 deletions(-)
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/ReadonlyConfig.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/ReadonlyConfig.java
index b4f6b31fe..ebf4d3853 100644
--- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/ReadonlyConfig.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/ReadonlyConfig.java
@@ -105,7 +105,7 @@ public class ReadonlyConfig implements Serializable {
if (value == null) {
return Optional.empty();
}
- return Optional.of(convertValue(value, option.typeReference()));
+ return Optional.of(convertValue(value, option));
}
@Override
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConfigUtil.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConfigUtil.java
index 426df5167..20541e1dc 100644
--- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConfigUtil.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConfigUtil.java
@@ -24,13 +24,20 @@ import org.apache.seatunnel.shade.com.fasterxml.jackson.dataformat.javaprop.Java
import org.apache.seatunnel.shade.com.typesafe.config.Config;
import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
+import org.apache.seatunnel.api.configuration.Option;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.lang.reflect.ParameterizedType;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.stream.Collectors;
+@Slf4j
public class ConfigUtil {
private static final JavaPropsMapper PROPERTIES_MAPPER = new JavaPropsMapper();
private static final ObjectMapper JACKSON_MAPPER = new ObjectMapper();
@@ -111,7 +118,8 @@ public class ConfigUtil {
}
@SuppressWarnings("unchecked")
- public static <T> T convertValue(Object rawValue, TypeReference<T> typeReference) {
+ public static <T> T convertValue(Object rawValue, Option<T> option) {
+ TypeReference<T> typeReference = option.typeReference();
rawValue = flatteningMapWithObject(rawValue);
if (typeReference.getType() instanceof Class) {
// simple type
@@ -129,6 +137,24 @@ public class ConfigUtil {
// complex type && untreated type
return JACKSON_MAPPER.readValue(convertToJsonString(rawValue), typeReference);
} catch (JsonProcessingException e) {
+ if (typeReference.getType() instanceof ParameterizedType
+ && List.class.equals(
+ ((ParameterizedType) typeReference.getType()).getRawType())) {
+ try {
+ log.warn(
+ String.format(
+ "Option '%s' is a List, and it is recommended to configure it as [\"string1\",\"string2\"]; we will only use ',' to split the String into a list.",
+ option.key()));
+ return (T)
+ convertToList(
+ rawValue,
+ (Class<T>)
+ ((ParameterizedType) typeReference.getType())
+ .getActualTypeArguments()[0]);
+ } catch (Exception ignore) {
+ // nothing
+ }
+ }
throw new IllegalArgumentException(
String.format(
"Json parsing exception, value '%s', and expected type '%s'",
@@ -137,6 +163,13 @@ public class ConfigUtil {
}
}
+ static <T> List<T> convertToList(Object rawValue, Class<T> clazz) {
+ return Arrays.stream(rawValue.toString().split(","))
+ .map(String::trim)
+ .map(value -> convertValue(value, clazz))
+ .collect(Collectors.toList());
+ }
+
@SuppressWarnings("unchecked")
static <T> T convertValue(Object rawValue, Class<T> clazz) {
if (Boolean.class.equals(clazz)) {
@@ -145,10 +178,76 @@ public class ConfigUtil {
return (T) convertToEnum(rawValue, (Class<? extends Enum<?>>) clazz);
} else if (String.class.equals(clazz)) {
return (T) convertToJsonString(rawValue);
+ } else if (Integer.class.equals(clazz)) {
+ return (T) convertToInt(rawValue);
+ } else if (Long.class.equals(clazz)) {
+ return (T) convertToLong(rawValue);
+ } else if (Float.class.equals(clazz)) {
+ return (T) convertToFloat(rawValue);
+ } else if (Double.class.equals(clazz)) {
+ return (T) convertToDouble(rawValue);
}
throw new IllegalArgumentException("Unsupported type: " + clazz);
}
+ static Integer convertToInt(Object o) {
+ if (o.getClass() == Integer.class) {
+ return (Integer) o;
+ } else if (o.getClass() == Long.class) {
+ long value = (Long) o;
+ if (value <= Integer.MAX_VALUE && value >= Integer.MIN_VALUE) {
+ return (int) value;
+ } else {
+ throw new IllegalArgumentException(
+ String.format(
+ "Configuration value %s overflows/underflows the integer type.",
+ value));
+ }
+ }
+
+ return Integer.parseInt(o.toString());
+ }
+
+ static Long convertToLong(Object o) {
+ if (o.getClass() == Long.class) {
+ return (Long) o;
+ } else if (o.getClass() == Integer.class) {
+ return ((Integer) o).longValue();
+ }
+
+ return Long.parseLong(o.toString());
+ }
+
+ static Float convertToFloat(Object o) {
+ if (o.getClass() == Float.class) {
+ return (Float) o;
+ } else if (o.getClass() == Double.class) {
+ double value = ((Double) o);
+ if (value == 0.0
+ || (value >= Float.MIN_VALUE && value <= Float.MAX_VALUE)
+ || (value >= -Float.MAX_VALUE && value <= -Float.MIN_VALUE)) {
+ return (float) value;
+ } else {
+ throw new IllegalArgumentException(
+ String.format(
+ "Configuration value %s overflows/underflows the float type.",
+ value));
+ }
+ }
+
+ return Float.parseFloat(o.toString());
+ }
+
+ static Double convertToDouble(Object o) {
+ if (o.getClass() == Double.class) {
+ return (Double) o;
+ } else if (o.getClass() == Float.class) {
+ return ((Float) o).doubleValue();
+ }
+
+ return Double.parseDouble(o.toString());
+ }
+
static Boolean convertToBoolean(Object o) {
switch (o.toString().toUpperCase()) {
case "TRUE":
diff --git a/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/ReadableConfigTest.java b/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/ReadableConfigTest.java
index 936f5b6f7..615d9d682 100644
--- a/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/ReadableConfigTest.java
+++ b/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/ReadableConfigTest.java
@@ -214,6 +214,11 @@ public class ReadableConfigTest {
list.add("VII");
Assertions.assertEquals(
list, config.get(Options.key("option.list").listType().noDefaultValue()));
+ list = new ArrayList<>();
+ list.add("Silk");
+ list.add("Song");
+ Assertions.assertEquals(
+ list, config.get(Options.key("option.list-str").listType().noDefaultValue()));
}
@Test
diff --git a/seatunnel-api/src/test/resources/conf/option-test.conf b/seatunnel-api/src/test/resources/conf/option-test.conf
index dc3b6b6de..de4a2b977 100644
--- a/seatunnel-api/src/test/resources/conf/option-test.conf
+++ b/seatunnel-api/src/test/resources/conf/option-test.conf
@@ -50,6 +50,7 @@ source {
option.enum = "LATEST"
option.list-json = """["Hello", "Apache SeaTunnel"]"""
option.list = ["final", "fantasy", "VII"]
+ option.list-str = "Silk,Song"
option.complex-type = [{
inner {
list = [{