You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ap...@apache.org on 2021/09/07 23:50:05 UTC
[pinot] branch config-obfuscator created (now fc7af0c)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a change to branch config-obfuscator
in repository https://gitbox.apache.org/repos/asf/pinot.git.
at fc7af0c basic obfuscator config
This branch includes the following new commits:
new fc7af0c basic obfuscator config
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[pinot] 01/01: basic obfuscator config
Posted by ap...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch config-obfuscator
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit fc7af0c8ea94d0aee364e85815b597c6be3dc01e
Author: Alexander Pucher <ap...@apache.org>
AuthorDate: Tue Sep 7 16:49:43 2021 -0700
basic obfuscator config
---
.../apache/pinot/spi/env/PinotConfiguration.java | 6 ++
.../org/apache/pinot/spi/utils/Obfuscator.java | 96 ++++++++++++++++++++++
.../apache/pinot/spi/config/ConfigUtilsTest.java | 17 ++++
.../org/apache/pinot/spi/utils/ObfuscatorTest.java | 83 +++++++++++++++++++
4 files changed, 202 insertions(+)
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/env/PinotConfiguration.java b/pinot-spi/src/main/java/org/apache/pinot/spi/env/PinotConfiguration.java
index ce4dd8f..e50b13f 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/env/PinotConfiguration.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/env/PinotConfiguration.java
@@ -32,6 +32,7 @@ import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.MapConfiguration;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.pinot.spi.ingestion.batch.spec.PinotFSSpec;
+import org.apache.pinot.spi.utils.Obfuscator;
/**
@@ -441,4 +442,9 @@ public class PinotConfiguration {
public Map<String, Object> toMap() {
return CommonsConfigurationUtils.toMap(_configuration);
}
+
+ @Override
+ public String toString() {
+ return String.valueOf(new Obfuscator().obfuscateJson(this));
+ }
}
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/Obfuscator.java b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/Obfuscator.java
new file mode 100644
index 0000000..7e0fd21
--- /dev/null
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/Obfuscator.java
@@ -0,0 +1,96 @@
+package org.apache.pinot.spi.utils;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.util.List;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+
+/**
+ * Simple obfuscator for object trees and configuration containers with key-value pairs. Matches a configurable set of
+ * patterns and replaces sensitive values with a pre-defined masked value for output.
+ *
+ * Example input:
+ * <pre>
+ * {
+ * "type": "sample object",
+ * "nestedCredentials": {
+ * "user": "admin",
+ * "password": "verysecret"
+ * }
+ * }
+ * </pre>
+ *
+ * Example output
+ * <pre>
+ * {
+ * "type": "sample object",
+ * "nestedCredentials": {
+ * "user": "admin",
+ * "password": "*****"
+ * }
+ * }
+ * </pre>
+ */
+public final class Obfuscator {
+ private static final String DEFAULT_MASKED_VALUE = "*****";
+ private static final List<Pattern> DEFAULT_PATTERNS =
+ Stream.of("(?i).*secret$", "(?i).*password$", "(?i).*token$").map(Pattern::compile).collect(Collectors.toList());
+
+ private final String _maskedValue;
+ private final List<Pattern> _patterns;
+
+ /**
+ * Obfuscator with default behavior matching (ignore case) "secret", "password", and "token" suffixes. Masks any
+ * values with '*****'
+ */
+ public Obfuscator() {
+ _maskedValue = DEFAULT_MASKED_VALUE;
+ _patterns = DEFAULT_PATTERNS;
+ }
+
+ /**
+ * Obfuscator with customized masking behavior. Defaults do not apply! Please ensure case-insensitive regex matching.
+ *
+ * @param maskedValue replacement value
+ * @param patterns key patterns to obfuscate
+ */
+ public Obfuscator(String maskedValue, List<Pattern> patterns) {
+ _maskedValue = maskedValue;
+ _patterns = patterns;
+ }
+
+ /**
+ * Serialize an object tree as JSON and obfuscate matching keys.
+ *
+ * @param object input value
+ * @return obfuscated JSON tree
+ */
+ public JsonNode obfuscateJson(Object object) {
+ // NOTE: jayway json path 2.4.0 seems to have issues with '@.name' so we'll do this manually
+ // as determined by a cursory and purely subjective investigation by alex
+ // "$..[?(@.name =~ /password$/i || @.name =~ /secret$/i || @.name =~ /token$/i)]"
+
+ return obfuscateJsonRec(JsonUtils.objectToJsonNode(object));
+ }
+
+ private JsonNode obfuscateJsonRec(JsonNode node) {
+ if (node.isObject()) {
+ node.fieldNames().forEachRemaining(field -> {
+ if (_patterns.stream().anyMatch(pattern -> pattern.matcher(field).matches())) {
+ ((ObjectNode) node).put(field, _maskedValue);
+ } else if (node.isArray()) {
+ IntStream.range(0, node.size()).forEach(i -> ((ArrayNode) node).set(i, obfuscateJsonRec(node.get(i))));
+ } else if (node.isObject()) {
+ ((ObjectNode) node).put(field, obfuscateJsonRec(node.get(field)));
+ }
+ });
+ }
+
+ return node;
+ }
+}
diff --git a/pinot-spi/src/test/java/org/apache/pinot/spi/config/ConfigUtilsTest.java b/pinot-spi/src/test/java/org/apache/pinot/spi/config/ConfigUtilsTest.java
index 5127357..61d9021 100644
--- a/pinot-spi/src/test/java/org/apache/pinot/spi/config/ConfigUtilsTest.java
+++ b/pinot-spi/src/test/java/org/apache/pinot/spi/config/ConfigUtilsTest.java
@@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.pinot.spi.config.table.IndexingConfig;
+import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.stream.OffsetCriteria;
import org.apache.pinot.spi.stream.StreamConfig;
import org.apache.pinot.spi.stream.StreamConfigProperties;
@@ -153,4 +154,20 @@ public class ConfigUtilsTest {
}
}
}
+
+ @Test
+ public void testDefaultObfuscation() {
+ Map<String, Object> map = new HashMap<>();
+ map.put("username", "admin");
+ map.put("password", "verysecret");
+ map.put("my.authToken", "secrettoken");
+
+ Map<String, Object> nestedMap = new HashMap<>();
+ map.put("credentials", map);
+
+ PinotConfiguration config = new PinotConfiguration(nestedMap);
+
+ Assert.assertFalse(config.toString().contains("verysecret"));
+ Assert.assertFalse(config.toString().contains("secrettoken"));
+ }
}
diff --git a/pinot-spi/src/test/java/org/apache/pinot/spi/utils/ObfuscatorTest.java b/pinot-spi/src/test/java/org/apache/pinot/spi/utils/ObfuscatorTest.java
new file mode 100644
index 0000000..204dd0b
--- /dev/null
+++ b/pinot-spi/src/test/java/org/apache/pinot/spi/utils/ObfuscatorTest.java
@@ -0,0 +1,83 @@
+package org.apache.pinot.spi.utils;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Pattern;
+import org.apache.commons.lang3.tuple.Pair;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+
+public class ObfuscatorTest {
+ private static final String VALUE = "VALUE";
+ private static final String SECRET = "SECRET";
+
+ private Obfuscator _obfuscator;
+
+ private Map<String, Object> _map;
+ private Map<String, Object> _nestedMap;
+
+ @BeforeMethod
+ public void setup() {
+ _obfuscator = new Obfuscator();
+
+ _map = new HashMap<>();
+ _map.put("value", "VALUE");
+ _map.put("secret", "SECRET");
+ _map.put("a.secret", "SECRET");
+ _map.put("mysecret", "SECRET");
+ _map.put("password", "SECRET");
+ _map.put("a.password", "SECRET");
+ _map.put("mypassword", "SECRET");
+ _map.put("token", "SECRET");
+ _map.put("a.token", "SECRET");
+ _map.put("mytoken", "SECRET");
+
+ _nestedMap = new HashMap<>();
+ _nestedMap.put("value", "VALUE");
+ _nestedMap.put("map", _map);
+ }
+
+ @Test
+ public void testSimple() {
+ String output = String.valueOf(_obfuscator.obfuscateJson(_map));
+ Assert.assertTrue(output.contains(VALUE));
+ Assert.assertFalse(output.contains(SECRET));
+ }
+
+ @Test
+ public void testNested() {
+ String output = String.valueOf(_obfuscator.obfuscateJson(_nestedMap));
+ Assert.assertTrue(output.contains(VALUE));
+ Assert.assertFalse(output.contains(SECRET));
+ }
+
+ @Test
+ public void testComplexObject() {
+ Object complex = Pair.of("nested", Pair.of("moreNested", Pair.of("mostNestedSecret", SECRET)));
+ String output = String.valueOf(_obfuscator.obfuscateJson(complex));
+ Assert.assertFalse(output.contains(SECRET));
+ }
+
+ @Test
+ public void testNull() {
+ Assert.assertEquals(String.valueOf(_obfuscator.obfuscateJson(null)), "null");
+ }
+
+ @Test
+ public void testNoop() {
+ Object output = new Obfuscator("nope", Collections.emptyList()).obfuscateJson(_map);
+ Assert.assertEquals(output, JsonUtils.objectToJsonNode(_map));
+ }
+
+ @Test
+ public void testCustomPattern() {
+ Obfuscator obfuscator = new Obfuscator("verycustomized", Collections.singletonList(Pattern.compile("^value$")));
+ String output = String.valueOf(obfuscator.obfuscateJson(_nestedMap));
+ Assert.assertFalse(output.contains(VALUE));
+ Assert.assertTrue(output.contains("verycustomized"));
+ Assert.assertTrue(output.contains(SECRET));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org