[pinot] 01/01: basic obfuscator config

[ap...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

apucher pushed a commit to branch config-obfuscator
in repository https://gitbox.apache.org/repos/asf/pinot.git

commit fc7af0c8ea94d0aee364e85815b597c6be3dc01e
Author: Alexander Pucher <ap...@apache.org>
AuthorDate: Tue Sep 7 16:49:43 2021 -0700

    basic obfuscator config
---
 .../apache/pinot/spi/env/PinotConfiguration.java   |  6 ++
 .../org/apache/pinot/spi/utils/Obfuscator.java     | 96 ++++++++++++++++++++++
 .../apache/pinot/spi/config/ConfigUtilsTest.java   | 17 ++++
 .../org/apache/pinot/spi/utils/ObfuscatorTest.java | 83 +++++++++++++++++++
 4 files changed, 202 insertions(+)

diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/env/PinotConfiguration.java b/pinot-spi/src/main/java/org/apache/pinot/spi/env/PinotConfiguration.java
index ce4dd8f..e50b13f 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/env/PinotConfiguration.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/env/PinotConfiguration.java
@@ -32,6 +32,7 @@ import org.apache.commons.configuration.ConfigurationException;
 import org.apache.commons.configuration.MapConfiguration;
 import org.apache.commons.configuration.PropertiesConfiguration;
 import org.apache.pinot.spi.ingestion.batch.spec.PinotFSSpec;
+import org.apache.pinot.spi.utils.Obfuscator;
 
 
 /**
@@ -441,4 +442,9 @@ public class PinotConfiguration {
   public Map<String, Object> toMap() {
     return CommonsConfigurationUtils.toMap(_configuration);
   }
+
+  @Override
+  public String toString() {
+    return String.valueOf(new Obfuscator().obfuscateJson(this));
+  }
 }
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/Obfuscator.java b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/Obfuscator.java
new file mode 100644
index 0000000..7e0fd21
--- /dev/null
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/Obfuscator.java
@@ -0,0 +1,96 @@
+package org.apache.pinot.spi.utils;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.util.List;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+
+/**
+ * Simple obfuscator for object trees and configuration containers with key-value pairs. Matches a configurable set of
+ * patterns and replaces sensitive values with a pre-defined masked value for output.
+ *
+ * Example input:
+ * <pre>
+ *   {
+ *     "type": "sample object",
+ *     "nestedCredentials": {
+ *       "user": "admin",
+ *       "password": "verysecret"
+ *     }
+ *   }
+ * </pre>
+ *
+ * Example output
+ * <pre>
+ *   {
+ *     "type": "sample object",
+ *     "nestedCredentials": {
+ *       "user": "admin",
+ *       "password": "*****"
+ *     }
+ *   }
+ * </pre>
+ */
+public final class Obfuscator {
+  private static final String DEFAULT_MASKED_VALUE = "*****";
+  private static final List<Pattern> DEFAULT_PATTERNS =
+      Stream.of("(?i).*secret$", "(?i).*password$", "(?i).*token$").map(Pattern::compile).collect(Collectors.toList());
+
+  private final String _maskedValue;
+  private final List<Pattern> _patterns;
+
+  /**
+   * Obfuscator with default behavior matching (ignore case) "secret", "password", and "token" suffixes. Masks any
+   * values with '*****'
+   */
+  public Obfuscator() {
+    _maskedValue = DEFAULT_MASKED_VALUE;
+    _patterns = DEFAULT_PATTERNS;
+  }
+
+  /**
+   * Obfuscator with customized masking behavior. Defaults do not apply! Please ensure case-insensitive regex matching.
+   *
+   * @param maskedValue replacement value
+   * @param patterns key patterns to obfuscate
+   */
+  public Obfuscator(String maskedValue, List<Pattern> patterns) {
+    _maskedValue = maskedValue;
+    _patterns = patterns;
+  }
+
+  /**
+   * Serialize an object tree as JSON and obfuscate matching keys.
+   *
+   * @param object input value
+   * @return obfuscated JSON tree
+   */
+  public JsonNode obfuscateJson(Object object) {
+    // NOTE: jayway json path 2.4.0 seems to have issues with '@.name' so we'll do this manually
+    // as determined by a cursory and purely subjective investigation by alex
+    // "$..[?(@.name =~ /password$/i || @.name =~ /secret$/i || @.name =~ /token$/i)]"
+
+    return obfuscateJsonRec(JsonUtils.objectToJsonNode(object));
+  }
+
+  private JsonNode obfuscateJsonRec(JsonNode node) {
+    if (node.isObject()) {
+      node.fieldNames().forEachRemaining(field -> {
+        if (_patterns.stream().anyMatch(pattern -> pattern.matcher(field).matches())) {
+          ((ObjectNode) node).put(field, _maskedValue);
+        } else if (node.isArray()) {
+          IntStream.range(0, node.size()).forEach(i -> ((ArrayNode) node).set(i, obfuscateJsonRec(node.get(i))));
+        } else if (node.isObject()) {
+          ((ObjectNode) node).put(field, obfuscateJsonRec(node.get(field)));
+        }
+      });
+    }
+
+    return node;
+  }
+}
diff --git a/pinot-spi/src/test/java/org/apache/pinot/spi/config/ConfigUtilsTest.java b/pinot-spi/src/test/java/org/apache/pinot/spi/config/ConfigUtilsTest.java
index 5127357..61d9021 100644
--- a/pinot-spi/src/test/java/org/apache/pinot/spi/config/ConfigUtilsTest.java
+++ b/pinot-spi/src/test/java/org/apache/pinot/spi/config/ConfigUtilsTest.java
@@ -26,6 +26,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import org.apache.pinot.spi.config.table.IndexingConfig;
+import org.apache.pinot.spi.env.PinotConfiguration;
 import org.apache.pinot.spi.stream.OffsetCriteria;
 import org.apache.pinot.spi.stream.StreamConfig;
 import org.apache.pinot.spi.stream.StreamConfigProperties;
@@ -153,4 +154,20 @@ public class ConfigUtilsTest {
       }
     }
   }
+
+  @Test
+  public void testDefaultObfuscation() {
+    Map<String, Object> map = new HashMap<>();
+    map.put("username", "admin");
+    map.put("password", "verysecret");
+    map.put("my.authToken", "secrettoken");
+
+    Map<String, Object> nestedMap = new HashMap<>();
+    map.put("credentials", map);
+
+    PinotConfiguration config = new PinotConfiguration(nestedMap);
+
+    Assert.assertFalse(config.toString().contains("verysecret"));
+    Assert.assertFalse(config.toString().contains("secrettoken"));
+  }
 }
diff --git a/pinot-spi/src/test/java/org/apache/pinot/spi/utils/ObfuscatorTest.java b/pinot-spi/src/test/java/org/apache/pinot/spi/utils/ObfuscatorTest.java
new file mode 100644
index 0000000..204dd0b
--- /dev/null
+++ b/pinot-spi/src/test/java/org/apache/pinot/spi/utils/ObfuscatorTest.java
@@ -0,0 +1,83 @@
+package org.apache.pinot.spi.utils;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Pattern;
+import org.apache.commons.lang3.tuple.Pair;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+
+public class ObfuscatorTest {
+  private static final String VALUE = "VALUE";
+  private static final String SECRET = "SECRET";
+
+  private Obfuscator _obfuscator;
+
+  private Map<String, Object> _map;
+  private Map<String, Object> _nestedMap;
+
+  @BeforeMethod
+  public void setup() {
+    _obfuscator = new Obfuscator();
+
+    _map = new HashMap<>();
+    _map.put("value", "VALUE");
+    _map.put("secret", "SECRET");
+    _map.put("a.secret", "SECRET");
+    _map.put("mysecret", "SECRET");
+    _map.put("password", "SECRET");
+    _map.put("a.password", "SECRET");
+    _map.put("mypassword", "SECRET");
+    _map.put("token", "SECRET");
+    _map.put("a.token", "SECRET");
+    _map.put("mytoken", "SECRET");
+
+    _nestedMap = new HashMap<>();
+    _nestedMap.put("value", "VALUE");
+    _nestedMap.put("map", _map);
+  }
+
+  @Test
+  public void testSimple() {
+    String output = String.valueOf(_obfuscator.obfuscateJson(_map));
+    Assert.assertTrue(output.contains(VALUE));
+    Assert.assertFalse(output.contains(SECRET));
+  }
+
+  @Test
+  public void testNested() {
+    String output = String.valueOf(_obfuscator.obfuscateJson(_nestedMap));
+    Assert.assertTrue(output.contains(VALUE));
+    Assert.assertFalse(output.contains(SECRET));
+  }
+
+  @Test
+  public void testComplexObject() {
+    Object complex = Pair.of("nested", Pair.of("moreNested", Pair.of("mostNestedSecret", SECRET)));
+    String output = String.valueOf(_obfuscator.obfuscateJson(complex));
+    Assert.assertFalse(output.contains(SECRET));
+  }
+
+  @Test
+  public void testNull() {
+    Assert.assertEquals(String.valueOf(_obfuscator.obfuscateJson(null)), "null");
+  }
+
+  @Test
+  public void testNoop() {
+    Object output = new Obfuscator("nope", Collections.emptyList()).obfuscateJson(_map);
+    Assert.assertEquals(output, JsonUtils.objectToJsonNode(_map));
+  }
+
+  @Test
+  public void testCustomPattern() {
+    Obfuscator obfuscator = new Obfuscator("verycustomized", Collections.singletonList(Pattern.compile("^value$")));
+    String output = String.valueOf(obfuscator.obfuscateJson(_nestedMap));
+    Assert.assertFalse(output.contains(VALUE));
+    Assert.assertTrue(output.contains("verycustomized"));
+    Assert.assertTrue(output.contains(SECRET));
+  }
+}

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org