You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2015/09/29 20:19:43 UTC
sqoop git commit: SQOOP-2549: Sqoop2: Allow connector developers to
mask certain keys in Maps when sending them back to clients
Repository: sqoop
Updated Branches:
refs/heads/sqoop2 bec7bd046 -> 2e74a9155
SQOOP-2549: Sqoop2: Allow connector developers to mask certain keys in Maps when sending them back to clients
(Abraham Fine via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/2e74a915
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/2e74a915
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/2e74a915
Branch: refs/heads/sqoop2
Commit: 2e74a9155ded8d2d4bdab14adb6270edde224b8f
Parents: bec7bd0
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Tue Sep 29 11:19:09 2015 -0700
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Tue Sep 29 11:19:09 2015 -0700
----------------------------------------------------------------------
.../sqoop/json/util/ConfigInputConstants.java | 1 +
.../json/util/ConfigInputSerialization.java | 11 ++++-
.../org/apache/sqoop/model/ConfigUtils.java | 3 +-
.../main/java/org/apache/sqoop/model/Input.java | 12 +++++-
.../java/org/apache/sqoop/model/MMapInput.java | 30 +++++++++++++-
.../json/util/TestConfigSerialization.java | 30 ++++++++++++--
.../org/apache/sqoop/model/TestConfigUtils.java | 2 +-
.../org/apache/sqoop/model/TestMConfig.java | 2 +-
.../org/apache/sqoop/model/TestMConfigList.java | 2 +-
.../java/org/apache/sqoop/model/TestMJob.java | 4 +-
.../org/apache/sqoop/model/TestMMapInput.java | 42 +++++++++++++++-----
docs/src/site/sphinx/ConnectorDevelopment.rst | 5 +++
.../common/CommonRepositoryHandler.java | 4 +-
.../sqoop/repository/derby/DerbyTestCase.java | 20 +++++-----
.../repository/mysql/MySqlTestCase.java | 4 +-
.../postgresql/PostgresqlTestCase.java | 4 +-
16 files changed, 136 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/common/src/main/java/org/apache/sqoop/json/util/ConfigInputConstants.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/util/ConfigInputConstants.java b/common/src/main/java/org/apache/sqoop/json/util/ConfigInputConstants.java
index 21739da..b301563 100644
--- a/common/src/main/java/org/apache/sqoop/json/util/ConfigInputConstants.java
+++ b/common/src/main/java/org/apache/sqoop/json/util/ConfigInputConstants.java
@@ -35,6 +35,7 @@ public class ConfigInputConstants {
public static final String CONFIG_INPUT_NAME = "name";
public static final String CONFIG_INPUT_TYPE = "type";
public static final String CONFIG_INPUT_SENSITIVE = "sensitive";
+ public static final String CONFIG_INPUT_SENSITIVE_KEY_PATTERN = "sensitive-pattern";
public static final String CONFIG_INPUT_SIZE = "size";
public static final String CONFIG_INPUT_EDITABLE = "editable";
public static final String CONFIG_INPUT_OVERRIDES = "overrides";
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/common/src/main/java/org/apache/sqoop/json/util/ConfigInputSerialization.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/util/ConfigInputSerialization.java b/common/src/main/java/org/apache/sqoop/json/util/ConfigInputSerialization.java
index eb42da3..03e869c 100644
--- a/common/src/main/java/org/apache/sqoop/json/util/ConfigInputSerialization.java
+++ b/common/src/main/java/org/apache/sqoop/json/util/ConfigInputSerialization.java
@@ -106,7 +106,13 @@ public final class ConfigInputSerialization {
// Skip if sensitive
if (!mInput.isEmpty() && !(skipSensitive && mInput.isSensitive())) {
if (mInput.getType() == MInputType.MAP) {
- input.put(ConfigInputConstants.CONFIG_INPUT_VALUE, mInput.getValue());
+ MMapInput mMapInput = (MMapInput)mInput;
+ input.put(ConfigInputConstants.CONFIG_INPUT_SENSITIVE_KEY_PATTERN, mMapInput.getSensitiveKeyPattern());
+ if (skipSensitive) {
+ input.put(ConfigInputConstants.CONFIG_INPUT_VALUE, mMapInput.getNonsenstiveValue());
+ } else {
+ input.put(ConfigInputConstants.CONFIG_INPUT_VALUE, mMapInput.getValue());
+ }
} else {
input.put(ConfigInputConstants.CONFIG_INPUT_VALUE, mInput.getUrlSafeValueString());
}
@@ -155,6 +161,7 @@ public final class ConfigInputSerialization {
InputEditable.valueOf((String)input.get(ConfigInputConstants.CONFIG_INPUT_EDITABLE))
: InputEditable.USER_ONLY;
String overrides = (String) input.get(ConfigInputConstants.CONFIG_INPUT_OVERRIDES);
+ String sensitveKeyPattern = (String) input.get(ConfigInputConstants.CONFIG_INPUT_SENSITIVE_KEY_PATTERN);
MInput mInput = null;
switch (type) {
@@ -164,7 +171,7 @@ public final class ConfigInputSerialization {
break;
}
case MAP: {
- mInput = new MMapInput(name, sensitive.booleanValue(), editable, overrides);
+ mInput = new MMapInput(name, sensitive.booleanValue(), editable, overrides, sensitveKeyPattern);
break;
}
case INTEGER: {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/common/src/main/java/org/apache/sqoop/model/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/ConfigUtils.java b/common/src/main/java/org/apache/sqoop/model/ConfigUtils.java
index 282a024..e5d4f80 100644
--- a/common/src/main/java/org/apache/sqoop/model/ConfigUtils.java
+++ b/common/src/main/java/org/apache/sqoop/model/ConfigUtils.java
@@ -144,6 +144,7 @@ public class ConfigUtils {
short maxLen = inputAnnotation.size();
InputEditable editable = inputAnnotation.editable();
String overrides = inputAnnotation.overrides();
+ String sensitiveKeyPattern = inputAnnotation.sensitiveKeyPattern();
Class<?> type = field.getType();
MInput input;
@@ -158,7 +159,7 @@ public class ConfigUtils {
if (type == String.class) {
input = new MStringInput(inputName, sensitive, editable, overrides, maxLen);
} else if (type.isAssignableFrom(Map.class)) {
- input = new MMapInput(inputName, sensitive, editable, overrides);
+ input = new MMapInput(inputName, sensitive, editable, overrides, sensitiveKeyPattern);
} else if (type == Integer.class) {
input = new MIntegerInput(inputName, sensitive, editable, overrides);
} else if (type == Long.class) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/common/src/main/java/org/apache/sqoop/model/Input.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/Input.java b/common/src/main/java/org/apache/sqoop/model/Input.java
index 883abe6..fa7f6ac 100644
--- a/common/src/main/java/org/apache/sqoop/model/Input.java
+++ b/common/src/main/java/org/apache/sqoop/model/Input.java
@@ -17,6 +17,7 @@
*/
package org.apache.sqoop.model;
+import org.apache.commons.lang.StringUtils;
import org.apache.sqoop.classification.InterfaceAudience;
import org.apache.sqoop.classification.InterfaceStability;
@@ -43,6 +44,15 @@ public @interface Input {
boolean sensitive() default false;
/**
+ * If this input is a map, keys matching this regular expression will be
+ * treated as sensitive. Sqoop will ensure that values associated with the
+ * sensitive keys will not be easily accessible.
+ *
+ * @return The regular expression that matches sensitive fields
+ */
+ String sensitiveKeyPattern() default StringUtils.EMPTY;
+
+ /**
* Indicates the entity that can edit the input's values, all inputs are
* created/deleted only by the connector code, other entities do not have
* access to either create/delete an input
@@ -63,7 +73,7 @@ public @interface Input {
* separated list of other inputs in the config class. It validates the
* attribute value obeys the expected conditions
*/
- String overrides() default "";
+ String overrides() default StringUtils.EMPTY;
/**
* List of validators associated with this input.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/common/src/main/java/org/apache/sqoop/model/MMapInput.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/model/MMapInput.java b/common/src/main/java/org/apache/sqoop/model/MMapInput.java
index f2c1ed7..4e8df34 100644
--- a/common/src/main/java/org/apache/sqoop/model/MMapInput.java
+++ b/common/src/main/java/org/apache/sqoop/model/MMapInput.java
@@ -20,7 +20,9 @@ package org.apache.sqoop.model;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
+import java.util.regex.Pattern;
+import org.apache.commons.lang.StringUtils;
import org.apache.sqoop.classification.InterfaceAudience;
import org.apache.sqoop.classification.InterfaceStability;
import org.apache.sqoop.utils.UrlSafeUtils;
@@ -29,8 +31,13 @@ import org.apache.sqoop.utils.UrlSafeUtils;
@InterfaceStability.Unstable
public final class MMapInput extends MInput<Map<String, String>> {
- public MMapInput(String name, boolean sensitive, InputEditable editable, String overrides) {
+ public static final String SENSITIVE_VALUE_PLACEHOLDER = StringUtils.EMPTY;
+
+ private final String sensitiveKeyPattern;
+
+ public MMapInput(String name, boolean sensitive, InputEditable editable, String overrides, String sensitiveKeyPattern) {
super(name, sensitive, editable, overrides);
+ this.sensitiveKeyPattern = sensitiveKeyPattern;
}
@Override
@@ -117,7 +124,7 @@ public final class MMapInput extends MInput<Map<String, String>> {
@Override
public MMapInput clone(boolean cloneWithValue) {
- MMapInput copy = new MMapInput(getName(), isSensitive(), getEditable(), getOverrides());
+ MMapInput copy = new MMapInput(getName(), isSensitive(), getEditable(), getOverrides(), getSensitiveKeyPattern());
copy.setPersistenceId(getPersistenceId());
if(cloneWithValue && this.getValue() != null) {
Map<String, String> copyMap = new HashMap<String, String>();
@@ -129,4 +136,23 @@ public final class MMapInput extends MInput<Map<String, String>> {
}
return copy;
}
+
+ public String getSensitiveKeyPattern() {
+ return sensitiveKeyPattern;
+ }
+
+ public Map<String, String> getNonsenstiveValue() {
+ if (isEmpty()) return null;
+
+ Map<String, String> nonsensitveValue = new HashMap<>();
+ Pattern sensitivePattern = Pattern.compile(getSensitiveKeyPattern());
+ for (Map.Entry<String, String> entry : getValue().entrySet()) {
+ if (sensitivePattern.matcher(entry.getKey()).matches()){
+ nonsensitveValue.put(entry.getKey(), SENSITIVE_VALUE_PLACEHOLDER);
+ } else {
+ nonsensitveValue.put(entry.getKey(), entry.getValue());
+ }
+ }
+ return nonsensitveValue;
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java b/common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java
index ab4f258..b188c4c 100644
--- a/common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java
+++ b/common/src/test/java/org/apache/sqoop/json/util/TestConfigSerialization.java
@@ -117,13 +117,37 @@ public class TestConfigSerialization {
String serializedJson = jsonObject.toJSONString();
// Replace map value with a fake string to force exception
- String badSerializedJson = serializedJson.replace("{\"A\":\"B\"}", "\"nonsensical string\"");
+ String badSerializedJson = serializedJson.replace("{\"A\":\"B\"}",
+ "\"nonsensical string\"");
System.out.println(badSerializedJson);
JSONObject retrievedJson = JSONUtils.parse(badSerializedJson);
ConfigInputSerialization.restoreConfig(retrievedJson);
}
@Test
+ public void testMapDataTypeSensitiveKeys() {
+ MConfig config = getMapConfig();
+
+ // Inserted values
+ Map<String, String> map = new HashMap<String, String>();
+ map.put("A", "B");
+ config.getMapInput("Map").setValue(map);
+
+ // Serialize
+ JSONObject jsonObject = ConfigInputSerialization.extractConfig(config, MConfigType.JOB, true);
+ String serializedJson = jsonObject.toJSONString();
+
+ // Map with sensitive values redacted
+ Map<String, String> sensitiveMap = new HashMap<String, String>();
+ sensitiveMap.put("A", MMapInput.SENSITIVE_VALUE_PLACEHOLDER);
+
+ // Deserialize
+ JSONObject retrievedJson = JSONUtils.parse(serializedJson);
+ MConfig retrieved = ConfigInputSerialization.restoreConfig(retrievedJson);
+ assertEquals(sensitiveMap, retrieved.getMapInput("Map").getValue());
+ }
+
+ @Test
public void testInputEditableOptional() {
// Inserted values
Map<String, String> map = new HashMap<String, String>();
@@ -188,7 +212,7 @@ public class TestConfigSerialization {
inputs = new LinkedList<MInput<?>>();
- input = new MMapInput("Map", false, InputEditable.ANY, StringUtils.EMPTY);
+ input = new MMapInput("Map", false, InputEditable.ANY, StringUtils.EMPTY, "A");
inputs.add(input);
return new MConfig("c", inputs);
@@ -208,7 +232,7 @@ public class TestConfigSerialization {
input = new MStringInput("String", false, InputEditable.ANY, StringUtils.EMPTY, (short)30);
inputs.add(input);
- input = new MMapInput("Map", false, InputEditable.ANY, StringUtils.EMPTY);
+ input = new MMapInput("Map", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY);
inputs.add(input);
input = new MIntegerInput("Integer", false, InputEditable.ANY, StringUtils.EMPTY);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/common/src/test/java/org/apache/sqoop/model/TestConfigUtils.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestConfigUtils.java b/common/src/test/java/org/apache/sqoop/model/TestConfigUtils.java
index 7052841..9ee0eec 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestConfigUtils.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestConfigUtils.java
@@ -245,7 +245,7 @@ public class TestConfigUtils {
// Config C
inputs = new LinkedList<MInput<?>>();
inputs.add(new MLongInput("cConfig.longValue", false, InputEditable.ANY, StringUtils.EMPTY));
- inputs.add(new MMapInput("cConfig.map", false, InputEditable.ANY, StringUtils.EMPTY));
+ inputs.add(new MMapInput("cConfig.map", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY));
inputs.add(new MEnumInput("cConfig.enumeration", false, InputEditable.ANY, StringUtils.EMPTY,
new String[] { "X", "Y" }));
inputs.add(new MListInput("cConfig.list", false, InputEditable.ANY, StringUtils.EMPTY));
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/common/src/test/java/org/apache/sqoop/model/TestMConfig.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMConfig.java b/common/src/test/java/org/apache/sqoop/model/TestMConfig.java
index f18b069..00b4a46 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMConfig.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMConfig.java
@@ -69,7 +69,7 @@ public class TestMConfig {
public void testGetInputs() {
MIntegerInput intInput = new MIntegerInput("Config.A", false, InputEditable.ANY, StringUtils.EMPTY );
MLongInput longInput = new MLongInput("Config.A1", false, InputEditable.ANY, StringUtils.EMPTY );
- MMapInput mapInput = new MMapInput("Config.B", false, InputEditable.ANY, StringUtils.EMPTY );
+ MMapInput mapInput = new MMapInput("Config.B", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY );
MStringInput stringInput = new MStringInput("Config.C", false, InputEditable.ANY,
StringUtils.EMPTY, (short) 3);
MEnumInput enumInput = new MEnumInput("Config.D", false, InputEditable.ANY, StringUtils.EMPTY,
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/common/src/test/java/org/apache/sqoop/model/TestMConfigList.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMConfigList.java b/common/src/test/java/org/apache/sqoop/model/TestMConfigList.java
index d0847fe..0d2ff61 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMConfigList.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMConfigList.java
@@ -32,7 +32,7 @@ public class TestMConfigList {
List<MConfig> configs = new LinkedList<MConfig>();
MIntegerInput intInput = new MIntegerInput("Config1.A", false, InputEditable.ANY, StringUtils.EMPTY);
- MMapInput mapInput = new MMapInput("Config1.B", false, InputEditable.ANY, StringUtils.EMPTY);
+ MMapInput mapInput = new MMapInput("Config1.B", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY);
List<MInput<?>> inputs = new ArrayList<MInput<?>>();
inputs.add(intInput);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/common/src/test/java/org/apache/sqoop/model/TestMJob.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMJob.java b/common/src/test/java/org/apache/sqoop/model/TestMJob.java
index e0ffdc2..6867ead 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMJob.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMJob.java
@@ -121,7 +121,7 @@ public class TestMJob {
private MToConfig toConfig() {
List<MConfig> configs = new ArrayList<MConfig>();
- MMapInput input = new MMapInput("MAP-INPUT", false, InputEditable.ANY, StringUtils.EMPTY);
+ MMapInput input = new MMapInput("MAP-INPUT", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY);
List<MInput<?>> list = new ArrayList<MInput<?>>();
list.add(input);
MConfig config = new MConfig("CONFIGTONAME", list);
@@ -131,7 +131,7 @@ public class TestMJob {
private MDriverConfig driverConfig() {
List<MConfig> configs = new ArrayList<MConfig>();
- MMapInput input = new MMapInput("MAP-INPUT", false, InputEditable.ANY, StringUtils.EMPTY);
+ MMapInput input = new MMapInput("MAP-INPUT", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY);
List<MInput<?>> list = new ArrayList<MInput<?>>();
list.add(input);
MConfig config = new MConfig("CONFIGDRIVERNAME", list);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/common/src/test/java/org/apache/sqoop/model/TestMMapInput.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/model/TestMMapInput.java b/common/src/test/java/org/apache/sqoop/model/TestMMapInput.java
index 118277e..6c2b7e6 100644
--- a/common/src/test/java/org/apache/sqoop/model/TestMMapInput.java
+++ b/common/src/test/java/org/apache/sqoop/model/TestMMapInput.java
@@ -38,7 +38,7 @@ public class TestMMapInput {
*/
@Test
public void testInitialization() {
- MMapInput input = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY);
+ MMapInput input = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY);
assertEquals("sqoopsqoop", input.getName());
assertEquals(MInputType.MAP, input.getType());
}
@@ -49,13 +49,13 @@ public class TestMMapInput {
@Test
public void testEquals() {
// Positive test
- MMapInput input1 = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY);
- MMapInput input2 = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY);
+ MMapInput input1 = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY);
+ MMapInput input2 = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY);
assertTrue(input1.equals(input2));
// Negative test
- MMapInput input3 = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY);
- MMapInput input4 = new MMapInput("sqoopsqoop1", false, InputEditable.ANY, StringUtils.EMPTY);
+ MMapInput input3 = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY);
+ MMapInput input4 = new MMapInput("sqoopsqoop1", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY);
assertFalse(input3.equals(input4));
}
@@ -64,7 +64,7 @@ public class TestMMapInput {
*/
@Test
public void testValue() {
- MMapInput input1 = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY);
+ MMapInput input1 = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY);
Map<String, String> map1 = new HashMap<String, String>();
input1.setValue(map1);
assertEquals(map1, input1.getValue());
@@ -77,7 +77,7 @@ public class TestMMapInput {
*/
@Test
public void testUrlSafe() {
- MMapInput input1 = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY);
+ MMapInput input1 = new MMapInput("sqoopsqoop", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY);
Map<String, String> map1 = new HashMap<String, String>();
input1.setValue(map1);
// Getting URL safe string
@@ -98,7 +98,7 @@ public class TestMMapInput {
*/
@Test
public void testNamedElement() {
- MMapInput input1 = new MMapInput("sqoopsqoop", true, InputEditable.ANY, StringUtils.EMPTY);
+ MMapInput input1 = new MMapInput("sqoopsqoop", true, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY);
assertEquals("sqoopsqoop.label", input1.getLabelKey());
assertEquals("sqoopsqoop.help", input1.getHelpKey());
}
@@ -108,9 +108,31 @@ public class TestMMapInput {
*/
@Test
public void testSensitivity() {
- MMapInput input1 = new MMapInput("NAME", false, InputEditable.ANY, StringUtils.EMPTY );
- MMapInput input2 = new MMapInput("NAME", true, InputEditable.ANY, StringUtils.EMPTY );
+ MMapInput input1 = new MMapInput("NAME", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY );
+ MMapInput input2 = new MMapInput("NAME", true, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY );
assertFalse(input1.isSensitive());
assertTrue(input2.isSensitive());
}
+
+ /**
+ * Test for sensitivity
+ */
+ @Test
+ public void testSensitiveKeyPattern() {
+ Map<String, String> testValue = new HashMap<>();
+ testValue.put("sqoop features", "awesome features");
+ testValue.put("sqoop bugs", "horrible bugs");
+
+ MMapInput input1 = new MMapInput("NAME", false, InputEditable.ANY, StringUtils.EMPTY, StringUtils.EMPTY );
+ input1.setValue(testValue);
+ MMapInput input2 = new MMapInput("NAME", true, InputEditable.ANY, StringUtils.EMPTY, ".*bugs.*");
+ input2.setValue(testValue);
+
+ assertEquals(input1.getNonsenstiveValue(), testValue);
+
+ Map<String, String> expectedNonsensitiveMap = new HashMap<>();
+ expectedNonsensitiveMap.put("sqoop features", "awesome features");
+ expectedNonsensitiveMap.put("sqoop bugs", MMapInput.SENSITIVE_VALUE_PLACEHOLDER);
+ assertEquals(input2.getNonsenstiveValue(), expectedNonsensitiveMap);
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/docs/src/site/sphinx/ConnectorDevelopment.rst
----------------------------------------------------------------------
diff --git a/docs/src/site/sphinx/ConnectorDevelopment.rst b/docs/src/site/sphinx/ConnectorDevelopment.rst
index b35c521..41389c8 100644
--- a/docs/src/site/sphinx/ConnectorDevelopment.rst
+++ b/docs/src/site/sphinx/ConnectorDevelopment.rst
@@ -401,6 +401,11 @@ Inputs associated with the link configuration include:
+-----------------------------+---------+-----------------------------------------------------------------------+-------------------------------------------------+
| sensitive | Boolean |Describes if the input value should be hidden from display |@Input(sensitive = true) public String password |
+-----------------------------+---------+-----------------------------------------------------------------------+-------------------------------------------------+
+| sensitiveKeyPattern | String |If the config paramteter is a map, this java regular expression |@Input(sensitiveKeyPattern = ".*sensitive") |
+| | |(http://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html)|public Map<String, String> sensitiveMap |
+| | |will be used to decide which keys are hidden from display. | |
+| | | | |
++-----------------------------+---------+-----------------------------------------------------------------------+-------------------------------------------------+
| editable | Enum |Describes the roles that can edit the value of this input |@Input(editable = ANY) public String value |
+-----------------------------+---------+-----------------------------------------------------------------------+-------------------------------------------------+
| overrides | String |Describes a list of other inputs this input can override in this config|@Input(overrides ="value") public String lvalue |
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java
----------------------------------------------------------------------
diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java
index 46b5272..79b9e99 100644
--- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java
+++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java
@@ -1998,7 +1998,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler {
input = new MStringInput(inputName, inputSensitivity, editableEnum, overrides, inputStrLength);
break;
case MAP:
- input = new MMapInput(inputName, inputSensitivity, editableEnum, overrides);
+ input = new MMapInput(inputName, inputSensitivity, editableEnum, overrides, StringUtils.EMPTY);
break;
case BOOLEAN:
input = new MBooleanInput(inputName, inputSensitivity, editableEnum, overrides);
@@ -2176,7 +2176,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler {
inputStrLength);
break;
case MAP:
- input = new MMapInput(inputName, inputSensitivity, editableEnum, overrides);
+ input = new MMapInput(inputName, inputSensitivity, editableEnum, overrides, StringUtils.EMPTY);
break;
case BOOLEAN:
input = new MBooleanInput(inputName, inputSensitivity, editableEnum, overrides);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
index e41fe38..ed176fe 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
@@ -1021,7 +1021,7 @@ abstract public class DerbyTestCase {
List<MInput<?>> inputs = new LinkedList<MInput<?>>();
MInput input = new MStringInput(configName1 + ".I1", false, InputEditable.USER_ONLY, configName1 + ".I2", (short) 30);
inputs.add(input);
- input = new MMapInput(configName1 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName1 + ".I5");
+ input = new MMapInput(configName1 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName1 + ".I5", StringUtils.EMPTY);
inputs.add(input);
input = new MIntegerInput(configName1 + ".I3", false, InputEditable.ANY, configName1 + ".I1");
inputs.add(input);
@@ -1034,7 +1034,7 @@ abstract public class DerbyTestCase {
inputs = new LinkedList<MInput<?>>();
input = new MStringInput(configName2 + ".I1", false, InputEditable.USER_ONLY, configName2 + ".I2", (short) 30);
inputs.add(input);
- input = new MMapInput(configName2 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName2 + ".I5");
+ input = new MMapInput(configName2 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName2 + ".I5", StringUtils.EMPTY);
inputs.add(input);
input = new MIntegerInput(configName2 + ".I3", false, InputEditable.ANY, configName2 + ".I1");
inputs.add(input);
@@ -1054,7 +1054,7 @@ abstract public class DerbyTestCase {
// I1 overrides another user_only attribute, hence a bad config
MInput input = new MStringInput(configName1 + ".I1", false, InputEditable.USER_ONLY, configName1 + ".I4", (short) 30);
inputs.add(input);
- input = new MMapInput(configName1 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName1 + ".I5");
+ input = new MMapInput(configName1 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName1 + ".I5", StringUtils.EMPTY);
inputs.add(input);
input = new MIntegerInput(configName1 + ".I3", false, InputEditable.ANY, configName1 + ".I1");
inputs.add(input);
@@ -1067,7 +1067,7 @@ abstract public class DerbyTestCase {
inputs = new LinkedList<MInput<?>>();
input = new MStringInput(configName2 + ".I1", false, InputEditable.USER_ONLY, configName2 + ".I2", (short) 30);
inputs.add(input);
- input = new MMapInput(configName2 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName2 + ".I5");
+ input = new MMapInput(configName2 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName2 + ".I5", StringUtils.EMPTY);
inputs.add(input);
input = new MIntegerInput(configName2 + ".I3", false, InputEditable.ANY, configName2 + ".I1");
inputs.add(input);
@@ -1087,7 +1087,7 @@ abstract public class DerbyTestCase {
// I1 overrides another user_only attribute, hence a bad config
MInput input = new MStringInput(configName1 + ".I1", false, InputEditable.USER_ONLY, configName1 + ".I4", (short) 30);
inputs.add(input);
- input = new MMapInput(configName1 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName1 + ".I5");
+ input = new MMapInput(configName1 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName1 + ".I5", StringUtils.EMPTY);
inputs.add(input);
input = new MIntegerInput(configName1 + ".I3", false, InputEditable.ANY, configName1 + ".I1");
inputs.add(input);
@@ -1100,7 +1100,7 @@ abstract public class DerbyTestCase {
inputs = new LinkedList<MInput<?>>();
input = new MStringInput(configName2 + ".I1", false, InputEditable.USER_ONLY, configName2 + ".I2", (short) 30);
inputs.add(input);
- input = new MMapInput(configName2 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName2 + ".I5");
+ input = new MMapInput(configName2 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName2 + ".I5", StringUtils.EMPTY);
inputs.add(input);
input = new MIntegerInput(configName2 + ".I3", false, InputEditable.ANY, configName2 + ".I1");
inputs.add(input);
@@ -1119,7 +1119,7 @@ abstract public class DerbyTestCase {
List<MInput<?>> inputs = new LinkedList<MInput<?>>();
MInput input = new MStringInput(configName1 + ".I1", false, InputEditable.USER_ONLY, configName1 + ".I2", (short) 30);
inputs.add(input);
- input = new MMapInput(configName1 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName1 + ".I1," + configName1 + ".I3");
+ input = new MMapInput(configName1 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName1 + ".I1," + configName1 + ".I3", StringUtils.EMPTY);
inputs.add(input);
input = new MIntegerInput(configName1 + ".I3", false, InputEditable.ANY, configName1 + ".I1");
inputs.add(input);
@@ -1132,7 +1132,7 @@ abstract public class DerbyTestCase {
inputs = new LinkedList<MInput<?>>();
input = new MStringInput(configName2 + ".I1", false, InputEditable.USER_ONLY, configName2 + ".I2", (short) 30);
inputs.add(input);
- input = new MMapInput(configName2 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName2 + ".I5");
+ input = new MMapInput(configName2 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName2 + ".I5", StringUtils.EMPTY);
inputs.add(input);
input = new MIntegerInput(configName2 + ".I3", false, InputEditable.ANY, configName2 + ".I1");
inputs.add(input);
@@ -1153,7 +1153,7 @@ abstract public class DerbyTestCase {
// I2 overrides a nonexistant input
MInput input = new MStringInput(configName1 + ".I1", false, InputEditable.USER_ONLY, configName1 + ".I2", (short) 30);
inputs.add(input);
- input = new MMapInput(configName1 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName1 + ".FOO");
+ input = new MMapInput(configName1 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName1 + ".FOO", StringUtils.EMPTY);
inputs.add(input);
input = new MIntegerInput(configName1 + ".I3", false, InputEditable.ANY, configName1 + ".I1");
inputs.add(input);
@@ -1166,7 +1166,7 @@ abstract public class DerbyTestCase {
inputs = new LinkedList<MInput<?>>();
input = new MStringInput(configName2 + ".I1", false, InputEditable.USER_ONLY, configName2 + ".I2", (short) 30);
inputs.add(input);
- input = new MMapInput(configName2 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName2 + ".I5");
+ input = new MMapInput(configName2 + ".I2", false, InputEditable.CONNECTOR_ONLY, configName2 + ".I5", StringUtils.EMPTY);
inputs.add(input);
input = new MIntegerInput(configName2 + ".I3", false, InputEditable.ANY, configName2 + ".I1");
inputs.add(input);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/repository/repository-mysql/src/test/java/org/apache/sqoop/integration/repository/mysql/MySqlTestCase.java
----------------------------------------------------------------------
diff --git a/repository/repository-mysql/src/test/java/org/apache/sqoop/integration/repository/mysql/MySqlTestCase.java b/repository/repository-mysql/src/test/java/org/apache/sqoop/integration/repository/mysql/MySqlTestCase.java
index 0bb3c63..44f968a 100644
--- a/repository/repository-mysql/src/test/java/org/apache/sqoop/integration/repository/mysql/MySqlTestCase.java
+++ b/repository/repository-mysql/src/test/java/org/apache/sqoop/integration/repository/mysql/MySqlTestCase.java
@@ -170,14 +170,14 @@ abstract public class MySqlTestCase extends TestCase {
MInput<?> input = new MStringInput("I1", false, InputEditable.ANY,
StringUtils.EMPTY, (short) 30);
inputs.add(input);
- input = new MMapInput("I2", false, InputEditable.ANY, "I1");
+ input = new MMapInput("I2", false, InputEditable.ANY, "I1", StringUtils.EMPTY);
inputs.add(input);
configs.add(new MConfig(configName1, inputs));
inputs = new LinkedList<MInput<?>>();
input = new MStringInput("I3", false, InputEditable.ANY, "I4", (short) 30);
inputs.add(input);
- input = new MMapInput("I4", false, InputEditable.ANY, "I3");
+ input = new MMapInput("I4", false, InputEditable.ANY, "I3", StringUtils.EMPTY);
inputs.add(input);
configs.add(new MConfig(configName2, inputs));
http://git-wip-us.apache.org/repos/asf/sqoop/blob/2e74a915/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java
----------------------------------------------------------------------
diff --git a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java
index 389ccec..faa0399 100644
--- a/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java
+++ b/repository/repository-postgresql/src/test/java/org/apache/sqoop/integration/repository/postgresql/PostgresqlTestCase.java
@@ -151,14 +151,14 @@ abstract public class PostgresqlTestCase {
List<MInput<?>> inputs = new LinkedList<MInput<?>>();
MInput<?> input = new MStringInput("I1", false, InputEditable.ANY, StringUtils.EMPTY, (short) 30);
inputs.add(input);
- input = new MMapInput("I2", false, InputEditable.ANY, "I1");
+ input = new MMapInput("I2", false, InputEditable.ANY, "I1", StringUtils.EMPTY);
inputs.add(input);
configs.add(new MConfig(configName1, inputs));
inputs = new LinkedList<MInput<?>>();
input = new MStringInput("I3", false, InputEditable.ANY, "I4", (short) 30);
inputs.add(input);
- input = new MMapInput("I4", false, InputEditable.ANY, "I3");
+ input = new MMapInput("I4", false, InputEditable.ANY, "I3", StringUtils.EMPTY);
inputs.add(input);
configs.add(new MConfig(configName2, inputs));