You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2015/05/21 19:29:44 UTC
ambari git commit: AMBARI-11289. Ranger HDFS/HBase Plugin Upgrade
Pack For HDP-2.2 To HDP-2.3 (ncole)
Repository: ambari
Updated Branches:
refs/heads/trunk 9371bb760 -> 8e3433c82
AMBARI-11289. Ranger HDFS/HBase Plugin Upgrade Pack For HDP-2.2 To HDP-2.3 (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8e3433c8
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8e3433c8
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8e3433c8
Branch: refs/heads/trunk
Commit: 8e3433c82063a0308596db37a940a6e264ca45de
Parents: 9371bb7
Author: Nate Cole <nc...@hortonworks.com>
Authored: Wed May 20 20:12:36 2015 -0400
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Thu May 21 13:29:30 2015 -0400
----------------------------------------------------------------------
.../serveraction/upgrades/ConfigureAction.java | 14 ++-
.../state/stack/upgrade/ConfigureTask.java | 17 ++-
.../stacks/HDP/2.2/upgrades/upgrade-2.3.xml | 112 ++++++++++++++++++-
.../upgrades/ConfigureActionTest.java | 4 +
4 files changed, 140 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/8e3433c8/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java
index c5f33c4..69a03f5 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java
@@ -275,7 +275,7 @@ public class ConfigureAction extends AbstractServerAction {
// append standard output
outputBuffer.append(MessageFormat.format("Created {0}/{1} = {2}\n", configType,
- transfer.toKey, valueToCopy));
+ transfer.toKey, mask(transfer, valueToCopy)));
}
break;
case MOVE:
@@ -295,7 +295,7 @@ public class ConfigureAction extends AbstractServerAction {
// append standard output
outputBuffer.append(MessageFormat.format("Created {0}/{1} with default value {2}\n",
- configType, transfer.toKey, transfer.defaultValue));
+ configType, transfer.toKey, mask(transfer, transfer.defaultValue)));
}
break;
@@ -370,7 +370,7 @@ public class ConfigureAction extends AbstractServerAction {
// without a key/value to set
newValues.put(key, value);
outputBuffer.append(MessageFormat.format("{0}/{1} changed to {2}\n", configType, key,
- value));
+ mask(keyValuePair, value)));
}
}
}
@@ -506,4 +506,12 @@ public class ConfigureAction extends AbstractServerAction {
return result;
}
+
+ private static String mask(ConfigureTask.Masked mask, String value) {
+ if (mask.mask) {
+ return StringUtils.repeat("*", value.length());
+ }
+ return value;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8e3433c8/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
index 99d6058..f5a77c5 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
@@ -154,11 +154,22 @@ public class ConfigureTask extends ServerSideActionTask {
}
/**
+ * Used for configuration updates that should mask their values from being
+ * printed in plain text.
+ */
+ @XmlAccessorType(XmlAccessType.FIELD)
+ public static class Masked {
+ @XmlAttribute(name = "mask")
+ public boolean mask = false;
+ }
+
+
+ /**
* A key/value pair to set in the type specified by {@link ConfigureTask#type}
*/
@XmlAccessorType(XmlAccessType.FIELD)
@XmlType(name = "set")
- public static class ConfigurationKeyValue {
+ public static class ConfigurationKeyValue extends Masked {
@XmlAttribute(name = "key")
public String key;
@@ -197,7 +208,7 @@ public class ConfigureTask extends ServerSideActionTask {
*/
@XmlAccessorType(XmlAccessType.FIELD)
@XmlType(name = "transfer")
- public static class Transfer {
+ public static class Transfer extends Masked {
/**
* The type of operation, such as COPY or DELETE.
*/
@@ -291,7 +302,7 @@ public class ConfigureTask extends ServerSideActionTask {
*/
@XmlAccessorType(XmlAccessType.FIELD)
@XmlType(name = "replace")
- public static class Replace {
+ public static class Replace extends Masked {
/**
* The key name
*/
http://git-wip-us.apache.org/repos/asf/ambari/blob/8e3433c8/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
index 7505a07..ddaee86 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
@@ -90,7 +90,6 @@
<service name="HBASE">
<component>HBASE_MASTER</component>
</service>
-
</group>
<group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check">
@@ -332,6 +331,60 @@
<value>org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer</value>
</condition>
</task>
+ <task xsi:type="configure" summary="Transitioning Ranger HDFS Policy">
+ <type>ranger-hdfs-policymgr-ssl</type>
+ <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="SSL_KEYSTORE_FILE_PATH" to-key="xasecure.policymgr.clientssl.keystore" default-value="/usr/hdp/current/hadoop-client/conf/ranger-plugin-keystore.jks" />
+ <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="SSL_KEYSTORE_PASSWORD" to-key="xasecure.policymgr.clientssl.keystore.password" mask="true" default-value="myKeyFilePassword" />
+ <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="SSL_TRUSTSTORE_FILE_PATH" to-key="xasecure.policymgr.clientssl.truststore" default-value="/usr/hdp/current/hadoop-client/conf/ranger-plugin-truststore.jks" />
+ <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="SSL_TRUSTSTORE_PASSWORD" to-key="xasecure.policymgr.clientssl.truststore.password" mask="true" default-value="changeit" />
+ </task>
+ <task xsi:type="configure" summary="Transitioning Ranger HDFS Audit">
+ <type>ranger-hdfs-audit</type>
+ <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="XAAUDIT.DB.IS_ENABLED" to-key="xasecure.audit.destination.db" default-value="false"/>
+ <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="XAAUDIT.HDFS.DESTINATION_DIRECTORY" to-key="xasecure.audit.destination.hdfs.dir" default-value="hdfs://{{namenode_hostname}}:8020/ranger/audit" />
+ <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="XAAUDIT.HDFS.IS_ENABLED" to-key="xasecure.audit.destination.hdfs" default-value="true" />
+ <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY" to-key="xasecure.audit.destination.hdfs.batch.filespool.dir" default-value="/var/log/hadoop/hdfs/audit/hdfs/spool" />
+ <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="XAAUDIT.DB.USER_NAME" to-key="xasecure.audit.destination.db.user" default-value="" />
+ <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="XAAUDIT.DB.PASSWORD" to-key="xasecure.audit.destination.db.password" mask="true" default-value="" />
+ <set key="xasecure.audit.destination.solr" value="true" />
+ <set key="xasecure.audit.destination.solr.urls" value="{{ranger_audit_solr_urls}}" />
+ <set key="xasecure.audit.destination.solr.zookeepers" value="" />
+ <set key="xasecure.audit.destination.solr.batch.filespool.dir" value="/var/logs/hdfs/audit/solr/spool" />
+ <set key="xasecure.audit.destination.db.jdbc.driver" value="{{jdbc_driver}}" />
+ <set key="xasecure.audit.destination.db.jdbc.url" value="{{audit_jdbc_url}}" />
+ <set key="xasecure.audit.provider.summary.enabled" value="true" />
+ </task>
+
+ <task xsi:type="configure" summary="Transitioning Ranger HDFS Security">
+ <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="REPOSITORY_NAME" to-key="ranger.plugin.hdfs.service.name" default-value="{{repo_name}}" />
+ <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="POLICY_MGR_URL" to-key="ranger.plugin.hdfs.policy.rest.url" default-value="{{policymgr_mgr_url}}" />
+ </task>
+
+ <task xsi:type="configure">
+ <type>ranger-hdfs-plugin-properties</type>
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_FILE" />
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS" />
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS" />
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS" />
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT" />
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY" />
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_FILE" />
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS" />
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS" />
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS" />
+ <transfer operation="delete" delete-key="SSL_KEYSTORE_FILE_PATH" />
+ <transfer operation="delete" delete-key="SSL_KEYSTORE_PASSWORD" />
+ <transfer operation="delete" delete-key="SSL_TRUSTSTORE_FILE_PATH" />
+ <transfer operation="delete" delete-key="SSL_TRUSTSTORE_PASSWORD" />
+ <transfer operation="delete" delete-key="XAAUDIT.DB.IS_ENABLED" />
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINATION_DIRECTORY" />
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.IS_ENABLED" />
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY" />
+ <transfer operation="delete" delete-key="XAAUDIT.DB.USER_NAME" />
+ <transfer operation="delete" delete-key="XAAUDIT.DB.PASSWORD" />
+ <transfer operation="delete" delete-key="REPOSITORY_NAME" />
+ <transfer operation="delete" delete-key="POLICY_MGR_URL" />
+ </task>
</pre-upgrade>
<upgrade>
<task xsi:type="restart" />
@@ -427,6 +480,63 @@
<service name="HBASE">
<component name="HBASE_MASTER">
+ <pre-upgrade>
+ <task xsi:type="configure" summary="Transitioning Ranger HBase Policy">
+ <type>ranger-hbase-policymgr-ssl</type>
+ <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="SSL_KEYSTORE_FILE_PATH" to-key="xasecure.policymgr.clientssl.keystore" default-value="/usr/hdp/current/hbase-client/conf/ranger-plugin-keystore.jks" />
+ <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="SSL_KEYSTORE_PASSWORD" to-key="xasecure.policymgr.clientssl.keystore.password" mask="true" default-value="myKeyFilePassword" />
+ <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="SSL_TRUSTSTORE_FILE_PATH" to-key="xasecure.policymgr.clientssl.truststore" default-value="/usr/hdp/current/hbase-client/conf/ranger-plugin-truststore.jks" />
+ <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="SSL_TRUSTSTORE_PASSWORD" to-key="xasecure.policymgr.clientssl.truststore.password" mask="true" default-value="changeit" />
+ </task>
+ <task xsi:type="configure" summary="Transitioning Ranger HBase Audit">
+ <type>ranger-hbase-audit</type>
+ <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="XAAUDIT.DB.IS_ENABLED" to-key="xasecure.audit.destination.db" default-value="false" />
+ <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="XAAUDIT.HDFS.DESTINATION_DIRECTORY" to-key="xasecure.audit.destination.hdfs.dir" default-value="hdfs://{{namenode_hostname}}:8020/ranger/audit" />
+ <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="XAAUDIT.HDFS.IS_ENABLED" to-key="xasecure.audit.destination.hdfs" default-value="true" />
+ <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY" to-key="xasecure.audit.destination.hdfs.batch.filespool.dir" default-value="/var/logs/hadoop/hdfs/audit/hdfs/spool" />
+ <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="XAAUDIT.DB.USER_NAME" to-key="xasecure.audit.destination.db.user" default-value="" />
+ <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="XAAUDIT.DB.PASSWORD" to-key="xasecure.audit.destination.db.password" mask="true" default-value="" />
+ <set key="xasecure.audit.destination.solr" value="true" />
+ <set key="xasecure.audit.destination.solr.urls" value="{{ranger_audit_solr_urls}}" />
+ <set key="xasecure.audit.destination.solr.zookeepers" value="" />
+ <set key="xasecure.audit.destination.solr.batch.filespool.dir" value="/var/logs/hbase/audit/solr/spool" />
+ <set key="xasecure.audit.destination.db.jdbc.driver" value="{{jdbc_driver}}" />
+ <set key="xasecure.audit.destination.db.jdbc.url" value="{{audit_jdbc_url}}" />
+ <set key="xasecure.audit.provider.summary.enabled" value="true" />
+ </task>
+ <task xsi:type="configure">
+ <type>ranger-hbase-security</type>
+ <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="UPDATE_XAPOLICIES_ON_GRANT_REVOKE" to-key="xasecure.hbase.update.xapolicies.on.grant.revoke" default-value="true" />
+ <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="POLICY_MGR_URL" to-key="ranger.plugin.hbase.policy.rest.url" default-value="{{policymgr_mgr_url}}" />
+ <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="REPOSITORY_NAME" to-key="ranger.plugin.hbase.service.name" default-value="{{repo_name}}" />
+ </task>
+ <task xsi:type="configure">
+ <type>ranger-hbase-plugin-properties</type>
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_FILE" />
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS" />
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS" />
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS" />
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT" />
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY" />
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_FILE" />
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS" />
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS" />
+ <transfer operation="delete" delete-key="SQL_CONNECTOR_JAR" />
+ <transfer operation="delete" delete-key="REPOSITORY_NAME" />
+ <transfer operation="delete" delete-key="POLICY_MGR_URL" />
+ <transfer operation="delete" delete-key="UPDATE_XAPOLICIES_ON_GRANT_REVOKE" />
+ <transfer operation="delete" delete-key="SSL_KEYSTORE_FILE_PATH" />
+ <transfer operation="delete" delete-key="SSL_KEYSTORE_PASSWORD" />
+ <transfer operation="delete" delete-key="SSL_TRUSTSTORE_FILE_PATH" />
+ <transfer operation="delete" delete-key="SSL_TRUSTSTORE_PASSWORD" />
+ <transfer operation="delete" delete-key="XAAUDIT.DB.IS_ENABLED" />
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINATION_DIRECTORY" />
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.IS_ENABLED" />
+ <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY" />
+ <transfer operation="delete" delete-key="XAAUDIT.DB.USER_NAME" />
+ <transfer operation="delete" delete-key="XAAUDIT.DB.PASSWORD" />
+ </task>
+ </pre-upgrade>
<upgrade>
<task xsi:type="restart" />
</upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/8e3433c8/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java
index c2ea948..08a8c35 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java
@@ -306,6 +306,7 @@ public class ConfigureActionTest {
transfer.fromKey = "movedFromKeyMissingWithDefault";
transfer.toKey = "movedToMissingWithDefault";
transfer.defaultValue = "defaultValue2";
+ transfer.mask = true;
transfers.add(transfer);
transfer = new ConfigureTask.Transfer();
@@ -544,6 +545,7 @@ public class ConfigureActionTest {
configurations.add(fooKey3);
fooKey3.key = "fooKey3";
fooKey3.value = "barValue3";
+ fooKey3.mask = true;
Map<String, String> commandParams = new HashMap<String, String>();
commandParams.put("upgrade_direction", "upgrade");
@@ -573,6 +575,8 @@ public class ConfigureActionTest {
assertEquals("barValue", config.getProperties().get("fooKey"));
assertEquals("barValue2", config.getProperties().get("fooKey2"));
assertEquals("barValue3", config.getProperties().get("fooKey3"));
+ assertTrue(report.getStdOut().contains("******"));
+
}
private void makeUpgradeCluster() throws Exception {