You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by je...@apache.org on 2019/06/02 05:35:30 UTC
[pulsar] branch master updated: add support for loading sink/source
secrets in inherited fields (#4430)
This is an automated email from the ASF dual-hosted git repository.
jerrypeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 4faccf1 add support for loading sink/source secrets in inherited fields (#4430)
4faccf1 is described below
commit 4faccf19fa11ec71021503f84f6a227ec9acf9e8
Author: Boyang Jerry Peng <je...@gmail.com>
AuthorDate: Sat Jun 1 22:35:24 2019 -0700
add support for loading sink/source secrets in inherited fields (#4430)
---
.../org/apache/pulsar/io/common/IOConfigUtils.java | 19 ++++-
.../apache/pulsar/io/common/IOConfigUtilsTest.java | 82 ++++++++++++++++++++--
2 files changed, 94 insertions(+), 7 deletions(-)
diff --git a/pulsar-io/common/src/main/java/org/apache/pulsar/io/common/IOConfigUtils.java b/pulsar-io/common/src/main/java/org/apache/pulsar/io/common/IOConfigUtils.java
index 95fd977..8066ea6 100644
--- a/pulsar-io/common/src/main/java/org/apache/pulsar/io/common/IOConfigUtils.java
+++ b/pulsar-io/common/src/main/java/org/apache/pulsar/io/common/IOConfigUtils.java
@@ -26,7 +26,10 @@ import org.apache.pulsar.io.core.annotations.FieldDoc;
import java.lang.annotation.Annotation;
import java.lang.reflect.Field;
+import java.util.Arrays;
import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
import java.util.function.Function;
@@ -40,14 +43,25 @@ public class IOConfigUtils {
return loadWithSecrets(map, clazz, secretName -> sinkContext.getSecret(secretName));
}
+
+ public static List<Field> getAllFields(Class<?> type) {
+ List<Field> fields = new LinkedList<>();
+ fields.addAll(Arrays.asList(type.getDeclaredFields()));
+
+ if (type.getSuperclass() != null) {
+ fields.addAll(getAllFields(type.getSuperclass()));
+ }
+
+ return fields;
+ }
+
private static <T> T loadWithSecrets(Map<String, Object> map, Class<T> clazz, Function<String, String> secretsGetter) {
Map<String, Object> configs = new HashMap<>(map);
- for (Field field : clazz.getDeclaredFields()) {
+ for (Field field : getAllFields(clazz)) {
field.setAccessible(true);
for (Annotation annotation : field.getAnnotations()) {
if (annotation.annotationType().equals(FieldDoc.class)) {
-
if (((FieldDoc) annotation).sensitive()) {
String secret = null;
try {
@@ -56,7 +70,6 @@ public class IOConfigUtils {
log.warn("Failed to read secret {}", field.getName(), e);
break;
}
-
if (secret != null) {
configs.put(field.getName(), secret);
}
diff --git a/pulsar-io/common/src/test/java/org/apache/pulsar/io/common/IOConfigUtilsTest.java b/pulsar-io/common/src/test/java/org/apache/pulsar/io/common/IOConfigUtilsTest.java
index 2296dae..df263de 100644
--- a/pulsar-io/common/src/test/java/org/apache/pulsar/io/common/IOConfigUtilsTest.java
+++ b/pulsar-io/common/src/test/java/org/apache/pulsar/io/common/IOConfigUtilsTest.java
@@ -19,6 +19,7 @@
package org.apache.pulsar.io.common;
import lombok.Data;
+import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.io.core.SinkContext;
import org.apache.pulsar.io.core.SourceContext;
@@ -44,7 +45,7 @@ public class IOConfigUtilsTest {
sensitive = true,
help = "password"
)
- private String password;
+ protected String password;
@FieldDoc(
required = true,
@@ -52,7 +53,7 @@ public class IOConfigUtilsTest {
sensitive = false,
help = ""
)
- private String notSensitive;
+ protected String notSensitive;
/**
* Non-string secrets are not supported at this moment
@@ -63,7 +64,29 @@ public class IOConfigUtilsTest {
sensitive = true,
help = ""
)
- private long sensitiveLong;
+ protected long sensitiveLong;
+ }
+
+ @Data
+ static class DerivedConfig extends TestConfig {
+ @FieldDoc(
+ required = true,
+ defaultValue = "",
+ sensitive = true,
+ help = ""
+ )
+ protected String moreSensitiveStuff;
+ }
+
+ @Data
+ static class DerivedDerivedConfig extends DerivedConfig {
+ @FieldDoc(
+ required = true,
+ defaultValue = "",
+ sensitive = true,
+ help = ""
+ )
+ protected String derivedDerivedSensitive;
}
static class TestSourceContext implements SourceContext {
@@ -71,6 +94,8 @@ public class IOConfigUtilsTest {
static Map<String, String> secretsMap = new HashMap<>();
static {
secretsMap.put("password", "my-source-password");
+ secretsMap.put("moreSensitiveStuff", "more-sensitive-stuff");
+ secretsMap.put("derivedDerivedSensitive", "derived-derived-sensitive");
}
@Override
@@ -159,7 +184,6 @@ public class IOConfigUtilsTest {
@Test
public void testSourceLoadWithSecrets() {
-
Map<String, Object> configMap = new HashMap<>();
configMap.put("notSensitive", "foo");
TestConfig testConfig = IOConfigUtils.loadWithSecrets(configMap, TestConfig.class, new TestSourceContext());
@@ -177,12 +201,38 @@ public class IOConfigUtilsTest {
Assert.assertEquals(testConfig.notSensitive, "foo");
Assert.assertEquals(testConfig.password, "my-source-password");
Assert.assertEquals(testConfig.sensitiveLong, 5L);
+
+ // test derived classes
+ configMap = new HashMap<>();
+ configMap.put("notSensitive", "foo");
+ configMap.put("sensitiveLong", 5L);
+
+ DerivedConfig derivedConfig = IOConfigUtils.loadWithSecrets(configMap, DerivedConfig.class, new TestSourceContext());
+
+ Assert.assertEquals(derivedConfig.notSensitive, "foo");
+ Assert.assertEquals(derivedConfig.password, "my-source-password");
+ Assert.assertEquals(derivedConfig.sensitiveLong, 5L);
+ Assert.assertEquals(derivedConfig.moreSensitiveStuff, "more-sensitive-stuff");
+
+ configMap = new HashMap<>();
+ configMap.put("notSensitive", "foo");
+ configMap.put("sensitiveLong", 5L);
+
+ DerivedDerivedConfig derivedDerivedConfig = IOConfigUtils.loadWithSecrets(configMap, DerivedDerivedConfig.class, new TestSourceContext());
+
+ Assert.assertEquals(derivedDerivedConfig.notSensitive, "foo");
+ Assert.assertEquals(derivedDerivedConfig.password, "my-source-password");
+ Assert.assertEquals(derivedDerivedConfig.sensitiveLong, 5L);
+ Assert.assertEquals(derivedDerivedConfig.moreSensitiveStuff, "more-sensitive-stuff");
+ Assert.assertEquals(derivedDerivedConfig.derivedDerivedSensitive, "derived-derived-sensitive");
}
static class TestSinkContext implements SinkContext {
static Map<String, String> secretsMap = new HashMap<>();
static {
secretsMap.put("password", "my-sink-password");
+ secretsMap.put("moreSensitiveStuff", "more-sensitive-stuff");
+ secretsMap.put("derivedDerivedSensitive", "derived-derived-sensitive");
}
@Override
@@ -290,5 +340,29 @@ public class IOConfigUtilsTest {
Assert.assertEquals(testConfig.notSensitive, "foo");
Assert.assertEquals(testConfig.password, "my-sink-password");
Assert.assertEquals(testConfig.sensitiveLong, 5L);
+
+ // test derived classes
+ configMap = new HashMap<>();
+ configMap.put("notSensitive", "foo");
+ configMap.put("sensitiveLong", 5L);
+
+ DerivedConfig derivedConfig = IOConfigUtils.loadWithSecrets(configMap, DerivedConfig.class, new TestSinkContext());
+
+ Assert.assertEquals(derivedConfig.notSensitive, "foo");
+ Assert.assertEquals(derivedConfig.password, "my-sink-password");
+ Assert.assertEquals(derivedConfig.sensitiveLong, 5L);
+ Assert.assertEquals(derivedConfig.moreSensitiveStuff, "more-sensitive-stuff");
+
+ configMap = new HashMap<>();
+ configMap.put("notSensitive", "foo");
+ configMap.put("sensitiveLong", 5L);
+
+ DerivedDerivedConfig derivedDerivedConfig = IOConfigUtils.loadWithSecrets(configMap, DerivedDerivedConfig.class, new TestSinkContext());
+
+ Assert.assertEquals(derivedDerivedConfig.notSensitive, "foo");
+ Assert.assertEquals(derivedDerivedConfig.password, "my-sink-password");
+ Assert.assertEquals(derivedDerivedConfig.sensitiveLong, 5L);
+ Assert.assertEquals(derivedDerivedConfig.moreSensitiveStuff, "more-sensitive-stuff");
+ Assert.assertEquals(derivedDerivedConfig.derivedDerivedSensitive, "derived-derived-sensitive");
}
}