You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by wa...@apache.org on 2023/04/28 03:24:04 UTC
[incubator-seatunnel] branch dev updated: [feature][api] Option support fallback keys (#4685)
This is an automated email from the ASF dual-hosted git repository.
wanghailin pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new b491020f5 [feature][api] Option support fallback keys (#4685)
b491020f5 is described below
commit b491020f5e5cc1406469b428ecf01effeba4b5d2
Author: Zongwen Li <zo...@apache.org>
AuthorDate: Fri Apr 28 11:23:57 2023 +0800
[feature][api] Option support fallback keys (#4685)
---
.../apache/seatunnel/api/common/CommonOptions.java | 3 +-
.../apache/seatunnel/api/configuration/Option.java | 21 ++++++++++++--
.../api/configuration/ReadonlyConfig.java | 32 ++++++++++++++++++----
.../api/configuration/util/ConfigUtil.java | 5 ++--
.../api/table/catalog/CatalogTableUtil.java | 8 ++----
.../seatunnel/api/configuration/OptionTest.java | 6 ++++
.../api/configuration/ReadableConfigTest.java | 15 ++++++++++
7 files changed, 71 insertions(+), 19 deletions(-)
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/common/CommonOptions.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/common/CommonOptions.java
index 5b6f404cc..e8c6f7878 100644
--- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/common/CommonOptions.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/common/CommonOptions.java
@@ -27,7 +27,8 @@ public interface CommonOptions {
Options.key("factory")
.stringType()
.noDefaultValue()
- .withDescription("Identifier of the SPI factory class.");
+ .withDescription("Identifier of the SPI factory class.")
+ .withFallbackKeys("plugin_name");
Option<String> PLUGIN_NAME =
Options.key("plugin_name")
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/Option.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/Option.java
index 693a6c84c..a4affa0c4 100644
--- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/Option.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/Option.java
@@ -19,6 +19,11 @@ package org.apache.seatunnel.api.configuration;
import org.apache.seatunnel.shade.com.fasterxml.jackson.core.type.TypeReference;
+import lombok.Getter;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
import java.util.Objects;
public class Option<T> {
@@ -34,10 +39,13 @@ public class Option<T> {
/** The description for this option. */
String description = "";
+ @Getter private final List<String> fallbackKeys;
+
public Option(String key, TypeReference<T> typeReference, T defaultValue) {
this.key = key;
this.typeReference = typeReference;
this.defaultValue = defaultValue;
+ this.fallbackKeys = new ArrayList<>();
}
public String key() {
@@ -61,6 +69,11 @@ public class Option<T> {
return this;
}
+ public Option<T> withFallbackKeys(String... fallbackKeys) {
+ this.fallbackKeys.addAll(Arrays.asList(fallbackKeys));
+ return this;
+ }
+
@Override
public boolean equals(Object obj) {
if (this == obj) {
@@ -71,16 +84,18 @@ public class Option<T> {
}
Option<?> that = (Option<?>) obj;
return Objects.equals(this.key, that.key)
- && Objects.equals(this.defaultValue, that.defaultValue);
+ && Objects.equals(this.defaultValue, that.defaultValue)
+ && Objects.equals(this.fallbackKeys, that.fallbackKeys);
}
@Override
public int hashCode() {
- return Objects.hash(this.key, this.defaultValue);
+ return Objects.hash(this.key, this.defaultValue, this.fallbackKeys);
}
@Override
public String toString() {
- return String.format("Key: '%s', default: %s", key, defaultValue);
+ return String.format(
+ "Key: '%s', default: %s (fallback keys: %s)", key, defaultValue, fallbackKeys);
}
}
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/ReadonlyConfig.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/ReadonlyConfig.java
index ebf4d3853..64ff5a05d 100644
--- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/ReadonlyConfig.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/ReadonlyConfig.java
@@ -23,6 +23,8 @@ import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.seatunnel.shade.com.typesafe.config.Config;
import org.apache.seatunnel.shade.com.typesafe.config.ConfigRenderOptions;
+import lombok.extern.slf4j.Slf4j;
+
import java.io.Serializable;
import java.util.Collections;
import java.util.HashMap;
@@ -34,6 +36,7 @@ import static org.apache.seatunnel.api.configuration.util.ConfigUtil.convertValu
import static org.apache.seatunnel.api.configuration.util.ConfigUtil.flatteningMap;
import static org.apache.seatunnel.api.configuration.util.ConfigUtil.treeMap;
+@Slf4j
public class ReadonlyConfig implements Serializable {
private static final long serialVersionUID = 1L;
private static final ObjectMapper JACKSON_MAPPER = new ObjectMapper();
@@ -89,23 +92,40 @@ public class ReadonlyConfig implements Serializable {
if (option == null) {
throw new NullPointerException("Option not be null.");
}
- String[] keys = option.key().split("\\.");
+ Object value = getValue(option.key());
+ if (value == null) {
+ for (String fallbackKey : option.getFallbackKeys()) {
+ value = getValue(fallbackKey);
+ if (value != null) {
+ log.info(
+ "Config uses fallback configuration key '{}' instead of key '{}'",
+ fallbackKey,
+ option.key());
+ break;
+ }
+ }
+ }
+ if (value == null) {
+ return Optional.empty();
+ }
+ return Optional.of(convertValue(value, option));
+ }
+
+ private Object getValue(String key) {
+ String[] keys = key.split("\\.");
Map<String, Object> data = this.confData;
Object value = null;
for (int i = 0; i < keys.length; i++) {
value = data.get(keys[i]);
if (i < keys.length - 1) {
if (!(value instanceof Map)) {
- return Optional.empty();
+ return null;
} else {
data = (Map<String, Object>) value;
}
}
}
- if (value == null) {
- return Optional.empty();
- }
- return Optional.of(convertValue(value, option));
+ return value;
}
@Override
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConfigUtil.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConfigUtil.java
index 20541e1dc..c9bf4655e 100644
--- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConfigUtil.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/util/ConfigUtil.java
@@ -142,9 +142,8 @@ public class ConfigUtil {
((ParameterizedType) typeReference.getType()).getRawType())) {
try {
log.warn(
- String.format(
- "Option '%s' is a List, and it is recommended to configure it as [\"string1\",\"string2\"]; we will only use ',' to split the String into a list.",
- option.key()));
+ "Option '{}' is a List, and it is recommended to configure it as [\"string1\",\"string2\"]; we will only use ',' to split the String into a list.",
+ option.key());
return (T)
convertToList(
rawValue,
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/CatalogTableUtil.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/CatalogTableUtil.java
index 250e2217d..7894f2e13 100644
--- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/CatalogTableUtil.java
+++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/table/catalog/CatalogTableUtil.java
@@ -99,16 +99,12 @@ public class CatalogTableUtil implements Serializable {
ReadonlyConfig readonlyConfig = ReadonlyConfig.fromConfig(config);
Map<String, String> catalogOptions =
readonlyConfig.getOptional(CatalogOptions.CATALOG_OPTIONS).orElse(new HashMap<>());
- // TODO: fallback key
- String factoryId =
- catalogOptions.getOrDefault(
- CommonOptions.FACTORY_ID.key(),
- readonlyConfig.get(CommonOptions.PLUGIN_NAME));
+
Map<String, Object> catalogAllOptions = new HashMap<>();
catalogAllOptions.putAll(readonlyConfig.toMap());
catalogAllOptions.putAll(catalogOptions);
ReadonlyConfig catalogConfig = ReadonlyConfig.fromMap(catalogAllOptions);
-
+ String factoryId = catalogConfig.get(CommonOptions.FACTORY_ID);
// Highest priority: specified schema
Map<String, String> schemaMap = readonlyConfig.get(CatalogTableUtil.SCHEMA);
if (schemaMap != null && schemaMap.size() > 0) {
diff --git a/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/OptionTest.java b/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/OptionTest.java
index 80b945c67..76d57ce03 100644
--- a/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/OptionTest.java
+++ b/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/OptionTest.java
@@ -45,5 +45,11 @@ public class OptionTest {
Assertions.assertEquals(
TEST_MODE,
Options.key("option.mode").enumType(TestMode.class).defaultValue(TestMode.LATEST));
+ Assertions.assertEquals(
+ TEST_NUM.withFallbackKeys("option.numeric"),
+ Options.key("option.num")
+ .intType()
+ .defaultValue(100)
+ .withFallbackKeys("option.numeric"));
}
}
diff --git a/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/ReadableConfigTest.java b/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/ReadableConfigTest.java
index d49b5aceb..b1436edd0 100644
--- a/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/ReadableConfigTest.java
+++ b/seatunnel-api/src/test/java/org/apache/seatunnel/api/configuration/ReadableConfigTest.java
@@ -283,4 +283,19 @@ public class ReadableConfigTest {
.listType(Double.class)
.noDefaultValue()));
}
+
+ @Test
+ public void testFallbackKey() {
+ Map<String, Object> map = new HashMap<>();
+ map.put("user", "ashulin");
+ final Option<String> usernameOption =
+ Options.key("username").stringType().noDefaultValue().withFallbackKeys("user");
+ ReadonlyConfig readonlyConfig = ReadonlyConfig.fromMap(map);
+ Assertions.assertEquals("ashulin", readonlyConfig.get(usernameOption));
+ Assertions.assertNull(
+ readonlyConfig.get(Options.key("username").stringType().noDefaultValue()));
+ map.put("username", "ark");
+ readonlyConfig = ReadonlyConfig.fromMap(map);
+ Assertions.assertEquals("ark", readonlyConfig.get(usernameOption));
+ }
}