You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by cr...@apache.org on 2015/04/08 01:25:47 UTC
[15/36] samza git commit: SAMZA-589;
sanitize sensitive data beore displaying configs to UI
SAMZA-589; sanitize sensitive data beore displaying configs to UI
Project: http://git-wip-us.apache.org/repos/asf/samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/cad2c828
Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/cad2c828
Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/cad2c828
Branch: refs/heads/samza-sql
Commit: cad2c828692244d110e9e3a19c39b257c3040b4c
Parents: 8170149
Author: Tommy Becker <tw...@gmail.com>
Authored: Mon Mar 16 14:53:32 2015 -0700
Committer: Chris Riccomini <cr...@apache.org>
Committed: Mon Mar 16 14:53:32 2015 -0700
----------------------------------------------------------------------
build.gradle | 1 +
.../versioned/jobs/configuration.md | 5 ++++-
.../java/org/apache/samza/config/Config.java | 5 +++++
.../java/org/apache/samza/config/MapConfig.java | 21 +++++++++++++++++---
.../org/apache/samza/config/TestConfig.java | 21 +++++++++++++++++++-
.../webapp/ApplicationMasterRestServlet.scala | 2 +-
.../webapp/ApplicationMasterWebServlet.scala | 2 +-
7 files changed, 50 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/samza/blob/cad2c828/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index 113da2a..97de3a2 100644
--- a/build.gradle
+++ b/build.gradle
@@ -56,6 +56,7 @@ rat {
'**/.git/**',
'**/.gitignore',
'**/.gradle/**',
+ '**/.idea/**',
'**/.project',
'**/.settings/**',
'**/.DS_Store/**',
http://git-wip-us.apache.org/repos/asf/samza/blob/cad2c828/docs/learn/documentation/versioned/jobs/configuration.md
----------------------------------------------------------------------
diff --git a/docs/learn/documentation/versioned/jobs/configuration.md b/docs/learn/documentation/versioned/jobs/configuration.md
index e094f60..c0ebb69 100644
--- a/docs/learn/documentation/versioned/jobs/configuration.md
+++ b/docs/learn/documentation/versioned/jobs/configuration.md
@@ -58,6 +58,9 @@ Configuration keys that absolutely must be defined for a Samza job are:
### Configuration Keys
-A complete list of configuration keys can be found on the [Configuration Table](configuration-table.html) page.
+A complete list of configuration keys can be found on the [Configuration Table](configuration-table.html) page. Note
+that configuration keys prefixed with "sensitive." are treated specially, in that the values associated with such keys
+will be masked in logs and Samza's YARN ApplicationMaster UI. This is to prevent accidental disclosure only; no
+encryption is done.
## [Packaging »](packaging.html)
http://git-wip-us.apache.org/repos/asf/samza/blob/cad2c828/samza-api/src/main/java/org/apache/samza/config/Config.java
----------------------------------------------------------------------
diff --git a/samza-api/src/main/java/org/apache/samza/config/Config.java b/samza-api/src/main/java/org/apache/samza/config/Config.java
index 9f7ade0..2b99050 100644
--- a/samza-api/src/main/java/org/apache/samza/config/Config.java
+++ b/samza-api/src/main/java/org/apache/samza/config/Config.java
@@ -33,6 +33,9 @@ import java.util.regex.Pattern;
* Store and retrieve named, typed values as configuration for classes implementing this interface.
*/
public abstract class Config implements Map<String, String> {
+ public static final String SENSITIVE_PREFIX = "sensitive.";
+ public static final String SENSITIVE_MASK = "********";
+
public Config subset(String prefix) {
return subset(prefix, true);
}
@@ -221,6 +224,8 @@ public abstract class Config implements Map<String, String> {
}
}
+ public abstract Config sanitize();
+
public void clear() {
throw new ConfigException("Config is immutable.");
}
http://git-wip-us.apache.org/repos/asf/samza/blob/cad2c828/samza-api/src/main/java/org/apache/samza/config/MapConfig.java
----------------------------------------------------------------------
diff --git a/samza-api/src/main/java/org/apache/samza/config/MapConfig.java b/samza-api/src/main/java/org/apache/samza/config/MapConfig.java
index 1a83923..38d7424 100644
--- a/samza-api/src/main/java/org/apache/samza/config/MapConfig.java
+++ b/samza-api/src/main/java/org/apache/samza/config/MapConfig.java
@@ -31,11 +31,11 @@ import java.util.Set;
*/
public class MapConfig extends Config {
private final Map<String, String> map;
-
+
public MapConfig() {
this.map = Collections.emptyMap();
}
-
+
public MapConfig(Map<String, String> map) {
this(Collections.singletonList(map));
}
@@ -79,6 +79,21 @@ public class MapConfig extends Config {
}
@Override
+ public Config sanitize() {
+ return new MapConfig(sanitizeMap());
+ }
+
+ private Map<String, String> sanitizeMap() {
+ Map<String, String> sanitized = new HashMap<String, String>(map);
+ for (Entry<String, String> entry : sanitized.entrySet()) {
+ if (entry.getKey().startsWith(SENSITIVE_PREFIX)) {
+ entry.setValue(SENSITIVE_MASK);
+ }
+ }
+ return sanitized;
+ }
+
+ @Override
public int hashCode() {
final int prime = 31;
int result = 1;
@@ -105,6 +120,6 @@ public class MapConfig extends Config {
@Override
public String toString() {
- return map.toString();
+ return sanitizeMap().toString();
}
}
http://git-wip-us.apache.org/repos/asf/samza/blob/cad2c828/samza-api/src/test/java/org/apache/samza/config/TestConfig.java
----------------------------------------------------------------------
diff --git a/samza-api/src/test/java/org/apache/samza/config/TestConfig.java b/samza-api/src/test/java/org/apache/samza/config/TestConfig.java
index b4100c2..d9f378d 100644
--- a/samza-api/src/test/java/org/apache/samza/config/TestConfig.java
+++ b/samza-api/src/test/java/org/apache/samza/config/TestConfig.java
@@ -19,7 +19,7 @@
package org.apache.samza.config;
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
import java.util.Map;
import java.util.HashMap;
@@ -61,4 +61,23 @@ public class TestConfig {
Class c4 = getClass(mc.getLong("testkey", defaultLong));
assertEquals(Long.class, c4);
}
+
+ @Test
+ public void testSanitize() {
+ Map<String, String> m = new HashMap<String, String>() {{
+ put("key1", "value1");
+ put("key2", "value2");
+ put("sensitive.key3", "secret1");
+ put("sensitive.key4", "secret2");
+ }};
+
+ Config config = new MapConfig(m);
+ assertFalse(config.toString().contains("secret"));
+
+ Config sanitized = config.sanitize();
+ assertEquals("value1", sanitized.get("key1"));
+ assertEquals("value2", sanitized.get("key2"));
+ assertEquals(Config.SENSITIVE_MASK, sanitized.get("sensitive.key3"));
+ assertEquals(Config.SENSITIVE_MASK, sanitized.get("sensitive.key4"));
+ }
}
http://git-wip-us.apache.org/repos/asf/samza/blob/cad2c828/samza-yarn/src/main/scala/org/apache/samza/webapp/ApplicationMasterRestServlet.scala
----------------------------------------------------------------------
diff --git a/samza-yarn/src/main/scala/org/apache/samza/webapp/ApplicationMasterRestServlet.scala b/samza-yarn/src/main/scala/org/apache/samza/webapp/ApplicationMasterRestServlet.scala
index 0afee64..4c855bf 100644
--- a/samza-yarn/src/main/scala/org/apache/samza/webapp/ApplicationMasterRestServlet.scala
+++ b/samza-yarn/src/main/scala/org/apache/samza/webapp/ApplicationMasterRestServlet.scala
@@ -103,6 +103,6 @@ class ApplicationMasterRestServlet(config: Config, state: SamzaAppMasterState, r
}
get("/config") {
- jsonMapper.writeValueAsString(new HashMap[String, Object](config.toMap))
+ jsonMapper.writeValueAsString(new HashMap[String, Object](config.sanitize.toMap))
}
}
http://git-wip-us.apache.org/repos/asf/samza/blob/cad2c828/samza-yarn/src/main/scala/org/apache/samza/webapp/ApplicationMasterWebServlet.scala
----------------------------------------------------------------------
diff --git a/samza-yarn/src/main/scala/org/apache/samza/webapp/ApplicationMasterWebServlet.scala b/samza-yarn/src/main/scala/org/apache/samza/webapp/ApplicationMasterWebServlet.scala
index 734d9d2..7fd5122 100644
--- a/samza-yarn/src/main/scala/org/apache/samza/webapp/ApplicationMasterWebServlet.scala
+++ b/samza-yarn/src/main/scala/org/apache/samza/webapp/ApplicationMasterWebServlet.scala
@@ -37,7 +37,7 @@ class ApplicationMasterWebServlet(config: Config, state: SamzaAppMasterState) ex
get("/") {
layoutTemplate("/WEB-INF/views/index.scaml",
- "config" -> TreeMap(config.toMap.toArray: _*),
+ "config" -> TreeMap(config.sanitize.toMap.toArray: _*),
"state" -> state,
"rmHttpAddress" -> WebAppUtils.getRMWebAppURLWithScheme(yarnConfig))
}