You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by yu...@apache.org on 2015/10/28 19:43:58 UTC
ambari git commit: AMBARI-13582. Allow use passwords references in
custom actions. (vbrodetskyi via yusaku)
Repository: ambari
Updated Branches:
refs/heads/trunk 987373628 -> 11db1a425
AMBARI-13582. Allow use passwords references in custom actions. (vbrodetskyi via yusaku)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/11db1a42
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/11db1a42
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/11db1a42
Branch: refs/heads/trunk
Commit: 11db1a4258b78f22e5bd3892239246a9989ef4f6
Parents: 9873736
Author: Yusaku Sako <yu...@hortonworks.com>
Authored: Wed Oct 28 11:42:54 2015 -0700
Committer: Yusaku Sako <yu...@hortonworks.com>
Committed: Wed Oct 28 11:42:54 2015 -0700
----------------------------------------------------------------------
.../controller/AmbariActionExecutionHelper.java | 4 +
.../AmbariManagementControllerImpl.java | 27 ++++---
.../ambari/server/controller/AmbariServer.java | 2 +-
.../controller/ConfigurationResponse.java | 13 +---
.../internal/BlueprintResourceProvider.java | 49 +++++++++---
.../apache/ambari/server/state/StackInfo.java | 19 +++++
.../server/state/cluster/ClusterImpl.java | 14 +---
.../server/topology/BlueprintValidatorImpl.java | 27 +++++++
.../ambari/server/utils/SecretReference.java | 78 +++++++++++++++-----
.../AmbariManagementControllerTest.java | 24 ++++--
.../internal/BlueprintResourceProviderTest.java | 47 ++++++++++--
.../server/topology/BlueprintImplTest.java | 63 +++++++++++++++-
.../services/YARN/configuration/yarn-site.xml | 6 ++
13 files changed, 291 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
index d834731..215aca8 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
@@ -46,6 +46,7 @@ import org.apache.ambari.server.state.ServiceComponentHost;
import org.apache.ambari.server.state.ServiceInfo;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.svccomphost.ServiceComponentHostOpInProgressEvent;
+import org.apache.ambari.server.utils.SecretReference;
import org.apache.ambari.server.utils.StageUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
@@ -406,6 +407,9 @@ public class AmbariActionExecutionHelper {
}
roleParams.putAll(actionContext.getParameters());
+
+ SecretReference.replaceReferencesWithPasswords(roleParams, cluster);
+
if (componentInfo != null) {
roleParams.put(COMPONENT_CATEGORY, componentInfo.getCategory());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 3a04a90..8099c64 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -729,9 +729,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
String passwordPropertyValue = requestProperties.get(passwordProperty);
if (!SecretReference.isSecret(passwordPropertyValue))
continue;
- SecretReference ref = new SecretReference(passwordPropertyValue, passwordProperty, cluster);
- if (!ref.getClusterName().equals(request.getClusterName()))
- throw new AmbariException("Can not reference to different cluster in SECRET");
+ SecretReference ref = new SecretReference(passwordPropertyValue, cluster);
String refValue = ref.getValue();
requestProperties.put(passwordProperty, refValue);
}
@@ -932,9 +930,9 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
if (LOG.isDebugEnabled()) {
LOG.debug("Received a getClusters request"
- + ", clusterName=" + request.getClusterName()
- + ", clusterId=" + request.getClusterId()
- + ", stackInfo=" + request.getStackVersion());
+ + ", clusterName=" + request.getClusterName()
+ + ", clusterId=" + request.getClusterId()
+ + ", stackInfo=" + request.getStackVersion());
}
Cluster singleCluster = null;
@@ -1401,7 +1399,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
if (propertiesTypes.containsKey(PropertyType.PASSWORD) &&
propertiesTypes.get(PropertyType.PASSWORD).contains(propertyName)) {
if (SecretReference.isSecret(propertyValue)) {
- SecretReference ref = new SecretReference(propertyValue, propertyName, cluster);
+ SecretReference ref = new SecretReference(propertyValue, cluster);
requestConfigProperties.put(propertyName, ref.getValue());
}
}
@@ -3399,30 +3397,31 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
ExecuteCommandJson jsons = customCommandExecutionHelper.getCommandJson(actionExecContext, cluster, stackId);
String commandParamsForStage = jsons.getCommandParamsForStage();
+ Map<String, String> commandParamsStage = gson.fromJson(commandParamsForStage, new TypeToken<Map<String, String>>()
+ {}.getType());
// Ensure that the specified requestContext (if any) is set as the request context
if (!requestContext.isEmpty()) {
requestStageContainer.setRequestContext(requestContext);
}
+ // replace password references in requestProperties
+ SecretReference.replaceReferencesWithPasswords(commandParamsStage, cluster);
+
// If the request is to perform the Kerberos service check, set up the stages to
// ensure that the (cluster-level) smoke user principal and keytab is available on all hosts
boolean kerberosServiceCheck = Role.KERBEROS_SERVICE_CHECK.name().equals(actionRequest.getCommandName());
if (kerberosServiceCheck) {
// Parse the command parameters into a map so that additional values may be added to it
- Map<String, String> commandParamsStage = gson.fromJson(commandParamsForStage,
- new TypeToken<Map<String, String>>() {
- }.getType());
try {
requestStageContainer = kerberosHelper.createTestIdentity(cluster, commandParamsStage, requestStageContainer);
} catch (KerberosOperationException e) {
throw new IllegalArgumentException(e.getMessage(), e);
}
-
- // Recreate commandParamsForStage with the added values
- commandParamsForStage = gson.toJson(commandParamsStage);
}
+ commandParamsForStage = gson.toJson(commandParamsStage);
+
Stage stage = createNewStage(requestStageContainer.getLastStageId(), cluster, requestId, requestContext,
jsons.getClusterHostInfo(), commandParamsForStage, jsons.getHostParamsForStage());
@@ -3453,7 +3452,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
if (kerberosServiceCheck) {
// Parse the command parameters into a map so that existing values may be accessed and
// additional values may be added to it.
- Map<String, String> commandParamsStage = gson.fromJson(commandParamsForStage,
+ commandParamsStage = gson.fromJson(commandParamsForStage,
new TypeToken<Map<String, String>>() {
}.getType());
http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
index 12373f0..484cb39 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
@@ -686,7 +686,7 @@ public class AmbariServer {
StackDefinedPropertyProvider.init(injector);
AbstractControllerResourceProvider.init(injector.getInstance(ResourceProviderFactory.class));
BlueprintResourceProvider.init(injector.getInstance(BlueprintFactory.class),
- injector.getInstance(BlueprintDAO.class), injector.getInstance(Gson.class));
+ injector.getInstance(BlueprintDAO.class), injector.getInstance(Gson.class), ambariMetaInfo);
StackDependencyResourceProvider.init(ambariMetaInfo);
ClusterResourceProvider.init(injector.getInstance(TopologyManager.class),
injector.getInstance(TopologyRequestFactoryImpl.class));
http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationResponse.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationResponse.java
index 3ed9306..eef3474 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationResponse.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ConfigurationResponse.java
@@ -79,7 +79,7 @@ public class ConfigurationResponse {
this.configs = configs;
this.configAttributes = configAttributes;
this.propertiesTypes = propertiesTypes;
- stubPasswords();
+ SecretReference.replacePasswordsWithReferences(propertiesTypes, configs, type, version);
}
/**
@@ -215,15 +215,4 @@ public class ConfigurationResponse {
public void setPropertiesTypes(Map<PropertyInfo.PropertyType, Set<String>> propertiesTypes) {
this.propertiesTypes = propertiesTypes;
}
-
- private void stubPasswords(){
- if(propertiesTypes != null && propertiesTypes.containsKey(PropertyInfo.PropertyType.PASSWORD)) {
- for(String pwdPropertyName: propertiesTypes.get(PropertyInfo.PropertyType.PASSWORD)) {
- if(configs.containsKey(pwdPropertyName)){
- String stub = SecretReference.generateStub(clusterName, type, version);
- configs.put(pwdPropertyName, stub);
- }
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java
index 6cb6a74..5994094 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java
@@ -32,6 +32,7 @@ import java.util.Set;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.DuplicateResourceException;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.spi.NoSuchParentResourceException;
import org.apache.ambari.server.controller.spi.NoSuchResourceException;
@@ -51,11 +52,13 @@ import org.apache.ambari.server.orm.entities.HostGroupComponentEntity;
import org.apache.ambari.server.orm.entities.HostGroupEntity;
import org.apache.ambari.server.orm.entities.StackEntity;
import org.apache.ambari.server.stack.NoSuchStackException;
+import org.apache.ambari.server.state.*;
import org.apache.ambari.server.topology.Blueprint;
import org.apache.ambari.server.topology.BlueprintFactory;
import org.apache.ambari.server.topology.InvalidTopologyException;
import com.google.gson.Gson;
+import org.apache.ambari.server.utils.SecretReference;
/**
@@ -116,6 +119,10 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
*/
private static Gson jsonSerializer;
+ /**
+ * Used to get stack metainfo.
+ */
+ private static AmbariMetaInfo ambariMetaInfo;
// ----- Constructors ----------------------------------------------------
@@ -140,10 +147,11 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
* @param dao blueprint data access object
* @param gson json serializer
*/
- public static void init(BlueprintFactory factory, BlueprintDAO dao, Gson gson) {
+ public static void init(BlueprintFactory factory, BlueprintDAO dao, Gson gson, AmbariMetaInfo metaInfo) {
blueprintFactory = factory;
blueprintDAO = dao;
jsonSerializer = gson;
+ ambariMetaInfo = metaInfo;
}
// ----- ResourceProvider ------------------------------------------------
@@ -175,7 +183,7 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
//todo: continue to use dao/entity directly or use blueprint factory?
public Set<Resource> getResources(Request request, Predicate predicate)
throws SystemException, UnsupportedPropertyException,
- NoSuchResourceException, NoSuchParentResourceException {
+ NoSuchResourceException, NoSuchParentResourceException {
List<BlueprintEntity> results = null;
boolean applyPredicate = false;
@@ -242,8 +250,8 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
modifyResources(new Command<Void>() {
@Override
public Void invoke() throws AmbariException {
- blueprintDAO.removeByName(blueprintName);
- return null;
+ blueprintDAO.removeByName(blueprintName);
+ return null;
}
});
}
@@ -263,7 +271,7 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
*
* @return a new resource instance for the given blueprint entity
*/
- protected Resource toResource(BlueprintEntity entity, Set<String> requestedIds) {
+ protected Resource toResource(BlueprintEntity entity, Set<String> requestedIds) throws NoSuchResourceException {
StackEntity stackEntity = entity.getStack();
Resource resource = new ResourceImpl(Resource.Type.Blueprint);
setResourceProperty(resource, BLUEPRINT_NAME_PROPERTY_ID, entity.getBlueprintName(), requestedIds);
@@ -304,16 +312,39 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide
* @return list of configuration property maps
*/
List<Map<String, Map<String, Object>>> populateConfigurationList(
- Collection<? extends BlueprintConfiguration> configurations) {
+ Collection<? extends BlueprintConfiguration> configurations) throws NoSuchResourceException {
List<Map<String, Map<String, Object>>> listConfigurations = new ArrayList<Map<String, Map<String, Object>>>();
for (BlueprintConfiguration config : configurations) {
Map<String, Map<String, Object>> mapConfigurations = new HashMap<String, Map<String, Object>>();
Map<String, Object> configTypeDefinition = new HashMap<String, Object>();
String type = config.getType();
- Map<String, Object> properties = jsonSerializer.<Map<String, Object>>fromJson(
- config.getConfigData(), Map.class);
- configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties);
+
+ if(config instanceof BlueprintConfigEntity) {
+ Map<String, String> properties = jsonSerializer.<Map<String, String>>fromJson(
+ config.getConfigData(), Map.class);
+
+ StackEntity stack = ((BlueprintConfigEntity)config).getBlueprintEntity().getStack();
+ StackInfo metaInfoStack;
+
+ try {
+ metaInfoStack = ambariMetaInfo.getStack(stack.getStackName(), stack.getStackVersion());
+ } catch (AmbariException e) {
+ throw new NoSuchResourceException(e.getMessage());
+ }
+
+ Map<org.apache.ambari.server.state.PropertyInfo.PropertyType, Set<String>> propertiesTypes =
+ metaInfoStack.getConfigPropertiesTypes(type);
+
+ SecretReference.replacePasswordsWithReferences(propertiesTypes, properties, type, -1l);
+
+ configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties);
+ } else {
+ Map<String, Object> properties = jsonSerializer.<Map<String, Object>>fromJson(
+ config.getConfigData(), Map.class);
+ configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties);
+ }
+
Map<String, Map<String, String>> attributes = jsonSerializer.<Map<String, Map<String, String>>>fromJson(
config.getConfigAttributes(), Map.class);
if (attributes != null && !attributes.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
index e3ac3e0..2b9cd83 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java
@@ -436,4 +436,23 @@ public class StackInfo implements Comparable<StackInfo>, Validable{
}
return result;
}
+
+ public Map<PropertyInfo.PropertyType, Set<String>> getConfigPropertiesTypes(String configType) {
+ Map<PropertyInfo.PropertyType, Set<String>> propertiesTypes = new HashMap<>();
+ Collection<ServiceInfo> services = getServices();
+ for (ServiceInfo serviceInfo : services) {
+ for (PropertyInfo propertyInfo : serviceInfo.getProperties()) {
+ if (propertyInfo.getFilename().contains(configType) && !propertyInfo.getPropertyTypes().isEmpty()) {
+ Set<PropertyInfo.PropertyType> types = propertyInfo.getPropertyTypes();
+ for (PropertyInfo.PropertyType propertyType : types) {
+ if (!propertiesTypes.containsKey(propertyType))
+ propertiesTypes.put(propertyType, new HashSet<String>());
+ propertiesTypes.get(propertyType).add(propertyInfo.getName());
+ }
+ }
+ }
+ }
+ return propertiesTypes;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
index 279b31f..c216110 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
@@ -2942,19 +2942,7 @@ public class ClusterImpl implements Cluster {
try {
StackId stackId = this.getCurrentStackVersion();
StackInfo stackInfo = ambariMetaInfo.getStack(stackId.getStackName(), stackId.getStackVersion());
- Collection<ServiceInfo> services = stackInfo.getServices();
- for (ServiceInfo serviceInfo : services) {
- for (PropertyInfo propertyInfo : serviceInfo.getProperties()) {
- if (propertyInfo.getFilename().contains(configType) && !propertyInfo.getPropertyTypes().isEmpty()) {
- Set<PropertyInfo.PropertyType> types = propertyInfo.getPropertyTypes();
- for (PropertyInfo.PropertyType propertyType : types) {
- if (!propertiesTypes.containsKey(propertyType))
- propertiesTypes.put(propertyType, new HashSet<String>());
- propertiesTypes.get(propertyType).add(propertyInfo.getName());
- }
- }
- }
- }
+ propertiesTypes = stackInfo.getConfigPropertiesTypes(configType);
} catch (Exception e) {
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
index 9e8f163..1c293ee 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
@@ -21,6 +21,7 @@ package org.apache.ambari.server.topology;
import org.apache.ambari.server.controller.internal.Stack;
import org.apache.ambari.server.state.AutoDeployInfo;
import org.apache.ambari.server.state.DependencyInfo;
+import org.apache.ambari.server.utils.SecretReference;
import org.apache.ambari.server.utils.VersionUtils;
import java.util.Collection;
@@ -83,6 +84,32 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
// we don't want to include default stack properties so we can't just use hostGroup full properties
Map<String, Map<String, String>> clusterConfigurations = blueprint.getConfiguration().getProperties();
+ // we need to have real passwords, not references
+ if(clusterConfigurations != null) {
+ StringBuilder errorMessage = new StringBuilder();
+ boolean containsSecretReferences = false;
+ for (Map.Entry<String, Map<String, String>> configEntry : clusterConfigurations.entrySet()) {
+ String configType = configEntry.getKey();
+ if (configEntry.getValue() != null) {
+ for (Map.Entry<String, String> propertyEntry : configEntry.getValue().entrySet()) {
+ String propertyName = propertyEntry.getKey();
+ String propertyValue = propertyEntry.getValue();
+ if (propertyValue != null) {
+ if (SecretReference.isSecret(propertyValue)) {
+ errorMessage.append(" Config:" + configType + " Property:" + propertyName+"\n");
+ containsSecretReferences = true;
+ }
+ }
+ }
+ }
+ }
+ if(containsSecretReferences) {
+ throw new InvalidTopologyException("Secret references are not allowed in blueprints, " +
+ "replace following properties with real passwords:\n"+errorMessage.toString());
+ }
+ }
+
+
for (HostGroup hostGroup : blueprint.getHostGroups().values()) {
Collection<String> processedServices = new HashSet<String>();
Map<String, Collection<String>> allRequiredProperties = new HashMap<String, Collection<String>>();
http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/main/java/org/apache/ambari/server/utils/SecretReference.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/SecretReference.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/SecretReference.java
index 2b1aeae..d801975 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/utils/SecretReference.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/SecretReference.java
@@ -21,37 +21,36 @@ package org.apache.ambari.server.utils;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.PropertyInfo;
import java.util.Map;
+import java.util.Set;
public class SecretReference {
- private String clusterName;
+ private static final String secretPrefix = "SECRET";
private String configType;
private Long version;
private String value;
- private String reference;
- public SecretReference(String reference, String propertyName, Cluster cluster) throws AmbariException{
+ public SecretReference(String reference, Cluster cluster) throws AmbariException{
String[] values = reference.split(":");
- clusterName = values[1];
- configType = values[2];
- version = Long.valueOf(values[3]);
+
+ configType = values[1];
+ version = Long.valueOf(values[2]);
+
+ String propertyName = values[3];
+ String clusterName = cluster.getClusterName();
Config refConfig = cluster.getConfigByVersion(configType, version);
if(refConfig == null)
- throw new AmbariException(String.format("Cluster: %s does not contain ConfigType: %s ConfigVersion: %s",
- cluster.getClusterName(), configType, version));
+ throw new AmbariException(String.format("Error when parsing secret reference. Cluster: %s does not contain ConfigType: %s ConfigVersion: %s",
+ clusterName, configType, version));
Map<String, String> refProperties = refConfig.getProperties();
if(!refProperties.containsKey(propertyName))
- throw new AmbariException(String.format("Cluster: %s ConfigType: %s ConfigVersion: %s does not contain property '%s'",
- cluster.getClusterName(), configType, version, propertyName));
- this.value = refProperties.get(propertyName);
-
- this.reference = reference;
- }
+ throw new AmbariException(String.format("Error when parsing secret reference. Cluster: %s ConfigType: %s ConfigVersion: %s does not contain property '%s'",
+ clusterName, configType, version, propertyName));
- public String getClusterName() {
- return clusterName;
+ this.value = refProperties.get(propertyName);
}
public void setConfigType(String configType) {
@@ -68,10 +67,51 @@ public class SecretReference {
public static boolean isSecret(String value) {
String[] values = value.split(":");
- return values.length == 4 && values[0].equals("SECRET");
+ return values.length == 4 && values[0].equals(secretPrefix);
+ }
+
+ public static String generateStub(String configType, Long configVersion, String propertyName) {
+ return secretPrefix + ":" + configType + ":" + configVersion.toString() + ":" + propertyName;
+ }
+
+ /**
+ * Replace secret references with appropriate real passwords.
+ * @param targetMap map in which replacement will be performed
+ * @param cluster current cluster
+ * @throws AmbariException
+ */
+ public static void replaceReferencesWithPasswords(Map<String, String> targetMap, Cluster cluster)
+ throws AmbariException {
+ if(cluster != null) {
+ for (Map.Entry<String, String> propertyValueEntry : targetMap.entrySet()) {
+ String key = propertyValueEntry.getKey();
+ String value = propertyValueEntry.getValue();
+ if (value != null && SecretReference.isSecret(value)) {
+ SecretReference ref = new SecretReference(value, cluster);
+ targetMap.put(key, ref.getValue());
+ }
+ }
+ }
}
- public static String generateStub(String clusterName, String configType, Long configVersion) {
- return "SECRET:" + clusterName + ":" + configType + ":" + configVersion.toString();
+ /**
+ * Replace real passwords with secret references
+ * @param propertiesTypes map with properties types
+ * @param propertiesMap map with properties in which replacement will be performed
+ * @param configType configuration type
+ * @param configVersion configuration version
+ */
+ public static void replacePasswordsWithReferences(Map<PropertyInfo.PropertyType, Set<String>> propertiesTypes,
+ Map<String, String> propertiesMap,
+ String configType,
+ Long configVersion){
+ if(propertiesTypes != null && propertiesTypes.containsKey(PropertyInfo.PropertyType.PASSWORD)) {
+ for(String pwdPropertyName: propertiesTypes.get(PropertyInfo.PropertyType.PASSWORD)) {
+ if(propertiesMap.containsKey(pwdPropertyName)){
+ String stub = SecretReference.generateStub(configType, configVersion, pwdPropertyName);
+ propertiesMap.put(pwdPropertyName, stub);
+ }
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index af8c5e2..5de7737 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -4123,7 +4123,7 @@ public class AmbariManagementControllerTest {
Config config1 = cf.createNew(cluster, "global",
new HashMap<String, String>() {{
put("key1", "value1");
- }}, new HashMap<String, Map<String,String>>());
+ }}, new HashMap<String, Map<String, String>>());
config1.setTag("version1");
Config config2 = cf.createNew(cluster, "core-site",
@@ -4132,8 +4132,15 @@ public class AmbariManagementControllerTest {
}}, new HashMap<String, Map<String,String>>());
config2.setTag("version1");
+ Config config3 = cf.createNew(cluster, "yarn-site",
+ new HashMap<String, String>() {{
+ put("test.password", "supersecret");
+ }}, new HashMap<String, Map<String,String>>());
+ config3.setTag("version1");
+
cluster.addConfig(config1);
cluster.addConfig(config2);
+ cluster.addConfig(config3);
Service hdfs = cluster.addService("HDFS");
hdfs.persist();
@@ -4161,6 +4168,7 @@ public class AmbariManagementControllerTest {
Map<String, String> params = new HashMap<String, String>() {{
put("test", "test");
+ put("pwd", "SECRET:yarn-site:1:test.password");
}};
Map<String, String> requestProperties = new HashMap<String, String>();
@@ -4193,6 +4201,8 @@ public class AmbariManagementControllerTest {
Map<String, String> commandParametersStage = StageUtils.getGson().fromJson(stage.getCommandParamsStage(), type);
Assert.assertTrue(commandParametersStage.containsKey("test"));
+ Assert.assertTrue(commandParametersStage.containsKey("pwd"));
+ Assert.assertEquals(commandParametersStage.get("pwd"), "supersecret");
Assert.assertEquals("HDFS", cmd.getServiceName());
Assert.assertEquals("DATANODE", cmd.getComponentName());
Assert.assertNotNull(hostParametersStage.get("jdk_location"));
@@ -4233,6 +4243,8 @@ public class AmbariManagementControllerTest {
commandParametersStage = StageUtils.getGson().fromJson(stage.getCommandParamsStage(), type);
Assert.assertTrue(commandParametersStage.containsKey("test"));
+ Assert.assertTrue(commandParametersStage.containsKey("pwd"));
+ Assert.assertEquals(commandParametersStage.get("pwd"), "supersecret");
Assert.assertEquals("HDFS", cmd.getServiceName());
Assert.assertEquals("DATANODE", cmd.getComponentName());
Assert.assertEquals(requestProperties.get(REQUEST_CONTEXT_PROPERTY), response.getRequestContext());
@@ -10589,7 +10601,7 @@ public class AmbariManagementControllerTest {
"hdfs-site",
"version2",
new HashMap<String, String>(){{
- put("test.password", "SECRET:c1:hdfs-site:1");
+ put("test.password", "SECRET:hdfs-site:1:test.password");
put("new", "new");//need this to mark config as "changed"
}},
new HashMap<String, Map<String, String>>()
@@ -10614,7 +10626,7 @@ public class AmbariManagementControllerTest {
"hdfs-site",
"version3",
new HashMap<String, String>(){{
- put("test.password", "SECRET:c1:hdfs-site:666");
+ put("test.password", "SECRET:hdfs-site:666:test.password");
}},
new HashMap<String, Map<String, String>>()
);
@@ -10642,7 +10654,7 @@ public class AmbariManagementControllerTest {
"hdfs-site",
"version5",
new HashMap<String, String>(){{
- put("test.password", "SECRET:c1:hdfs-site:4");
+ put("test.password", "SECRET:hdfs-site:4:test.password");
put("new", "new");
}},
new HashMap<String, Map<String, String>>()
@@ -10653,7 +10665,7 @@ public class AmbariManagementControllerTest {
controller.updateClusters(Collections.singleton(crReq), null);
fail("Request need to be failed with wrong secret reference");
} catch (AmbariException e) {
- assertEquals("Cluster: foo1 ConfigType: hdfs-site ConfigVersion: 4 does not contain property 'test.password'",
+ assertEquals("Error when parsing secret reference. Cluster: foo1 ConfigType: hdfs-site ConfigVersion: 4 does not contain property 'test.password'",
e.getMessage());
}
cl.getAllConfigs();
@@ -10676,7 +10688,7 @@ public class AmbariManagementControllerTest {
add(configRequest);
}});
for(ConfigurationResponse resp : requestedConfigs) {
- String secretName = "SECRET:foo1:hdfs-site:"+resp.getVersion().toString();
+ String secretName = "SECRET:hdfs-site:"+resp.getVersion().toString()+":test.password";
if(resp.getConfigs().containsKey("test.password")) {
assertEquals(resp.getConfigs().get("test.password"), secretName);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java
index 5bfdebb..8ef4bbb 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java
@@ -44,6 +44,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.internal.BlueprintResourceProvider.BlueprintConfigPopulationStrategy;
@@ -69,6 +70,8 @@ import org.apache.ambari.server.orm.entities.HostGroupComponentEntity;
import org.apache.ambari.server.orm.entities.HostGroupConfigEntity;
import org.apache.ambari.server.orm.entities.HostGroupEntity;
import org.apache.ambari.server.orm.entities.StackEntity;
+import org.apache.ambari.server.state.*;
+import org.apache.ambari.server.state.PropertyInfo;
import org.apache.ambari.server.utils.StageUtils;
import org.apache.ambari.server.topology.Blueprint;
import org.apache.ambari.server.topology.BlueprintFactory;
@@ -104,7 +107,7 @@ public class BlueprintResourceProviderTest {
@BeforeClass
public static void initClass() {
- BlueprintResourceProvider.init(blueprintFactory, dao, gson);
+ BlueprintResourceProvider.init(blueprintFactory, dao, gson, metaInfo);
StackEntity stackEntity = new StackEntity();
stackEntity.setStackName("test-stack-name");
@@ -381,7 +384,12 @@ public class BlueprintResourceProviderTest {
@Test
public void testGetResourcesNoPredicate_withConfiguration() throws SystemException, UnsupportedPropertyException,
- NoSuchParentResourceException, NoSuchResourceException {
+ NoSuchParentResourceException, NoSuchResourceException, AmbariException {
+
+ StackInfo info = createMock(StackInfo.class);
+ expect(info.getConfigPropertiesTypes("core-site")).andReturn(new HashMap<PropertyInfo.PropertyType, Set<String>>()).anyTimes();
+ expect(metaInfo.getStack("test-stack-name", "test-stack-version")).andReturn(info).anyTimes();
+ replay(info, metaInfo);
Request request = createNiceMock(Request.class);
Set<Map<String, Object>> testProperties = getBlueprintTestProperties();
@@ -951,20 +959,44 @@ public class BlueprintResourceProviderTest {
@Test
public void testPopulateConfigurationList() throws Exception {
+ StackEntity stackEntity = new StackEntity();
+ stackEntity.setStackName("test-stack-name");
+ stackEntity.setStackVersion("test-stack-version");
+ BlueprintEntity entity = createMock(BlueprintEntity.class);
+ expect(entity.getStack()).andReturn(stackEntity).anyTimes();
+
+ HashMap<PropertyInfo.PropertyType, Set<String>> pwdProperties = new HashMap<PropertyInfo.PropertyType, Set<String>>() {{
+ put(PropertyInfo.PropertyType.PASSWORD, new HashSet<String>(){{
+ add("test.password");
+ }});
+ }};
+
+ StackInfo info = createMock(StackInfo.class);
+ expect(info.getConfigPropertiesTypes("type1")).andReturn(new HashMap<PropertyInfo.PropertyType, Set<String>>()).anyTimes();
+ expect(info.getConfigPropertiesTypes("type2")).andReturn(new HashMap<PropertyInfo.PropertyType, Set<String>>()).anyTimes();
+ expect(info.getConfigPropertiesTypes("type3")).andReturn(pwdProperties).anyTimes();
+ expect(metaInfo.getStack("test-stack-name", "test-stack-version")).andReturn(info).anyTimes();
+
+ replay(info, metaInfo, entity);
+
+
// attributes is null
- BlueprintConfiguration config1 = new BlueprintConfigEntity();
+ BlueprintConfigEntity config1 = new BlueprintConfigEntity();
config1.setType("type1");
config1.setConfigData("{\"key1\":\"value1\"}");
+ config1.setBlueprintEntity(entity);
// attributes is empty
- BlueprintConfiguration config2 = new BlueprintConfigEntity();
+ BlueprintConfigEntity config2 = new BlueprintConfigEntity();
config2.setType("type2");
config2.setConfigData("{\"key2\":\"value2\"}");
config2.setConfigAttributes("{}");
+ config2.setBlueprintEntity(entity);
// attributes is provided
- BlueprintConfiguration config3 = new BlueprintConfigEntity();
+ BlueprintConfigEntity config3 = new BlueprintConfigEntity();
config3.setType("type3");
- config3.setConfigData("{\"key3\":\"value3\",\"key4\":\"value4\"}");
+ config3.setConfigData("{\"key3\":\"value3\",\"key4\":\"value4\",\"test.password\":\"pwdValue\"}");
config3.setConfigAttributes("{\"final\":{\"key3\":\"attrValue1\",\"key4\":\"attrValue2\"}}");
+ config3.setBlueprintEntity(entity);
List<Map<String, Map<String, Object>>> configs =
provider.populateConfigurationList(Arrays.asList(config1, config2, config3));
@@ -1010,9 +1042,10 @@ public class BlueprintResourceProviderTest {
Map<String, String> confProperties3
= (Map<String, String>) typeConfig3.get(BlueprintResourceProvider.PROPERTIES_PROPERTY_ID);
assertNotNull(confProperties3);
- assertEquals(2, confProperties3.size());
+ assertEquals(3, confProperties3.size());
assertEquals("value3", confProperties3.get("key3"));
assertEquals("value4", confProperties3.get("key4"));
+ assertEquals("SECRET:type3:-1:test.password", confProperties3.get("test.password"));
assertTrue(typeConfig3.containsKey(BlueprintResourceProvider.PROPERTIES_ATTRIBUTES_PROPERTY_ID));
Map<String, Map<String, String>> confAttributes3
= (Map<String, Map<String, String>>) typeConfig3.get(BlueprintResourceProvider.PROPERTIES_ATTRIBUTES_PROPERTY_ID);
http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
index 9d4163a..de740f4 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
@@ -183,7 +183,6 @@ public class BlueprintImplTest {
properties.put("hdfs-site", hdfsProps);
hdfsProps.put("foo", "val");
hdfsProps.put("bar", "val");
-
Map<String, String> category1Props = new HashMap<String, String>();
properties.put("category1", category1Props);
category1Props.put("prop1", "val");
@@ -282,6 +281,68 @@ public class BlueprintImplTest {
verify(stack, group1, group2);
}
+ @Test
+ public void testValidateConfigurations__secretReference(){
+ Stack stack = createNiceMock(Stack.class);
+
+ HostGroup group1 = createNiceMock(HostGroup.class);
+ HostGroup group2 = createNiceMock(HostGroup.class);
+ Collection<HostGroup> hostGroups = new HashSet<HostGroup>();
+ hostGroups.add(group1);
+ hostGroups.add(group2);
+
+ Set<String> group1Components = new HashSet<String>();
+ group1Components.add("c1");
+ group1Components.add("c2");
+
+ Set<String> group2Components = new HashSet<String>();
+ group2Components.add("c1");
+ group2Components.add("c3");
+
+ Map<String, Map<String, String>> group2Props = new HashMap<String, Map<String, String>>();
+ Map<String, String> group2Category2Props = new HashMap<String, String>();
+ group2Props.put("category2", group2Category2Props);
+ group2Category2Props.put("prop2", "val");
+
+ Collection<Stack.ConfigProperty> requiredHDFSProperties = new HashSet<Stack.ConfigProperty>();
+ requiredHDFSProperties.add(new Stack.ConfigProperty("hdfs-site", "foo", null));
+ requiredHDFSProperties.add(new Stack.ConfigProperty("hdfs-site", "bar", null));
+ requiredHDFSProperties.add(new Stack.ConfigProperty("hdfs-site", "some_password", null));
+
+ requiredHDFSProperties.add(new Stack.ConfigProperty("category1", "prop1", null));
+
+ Collection<Stack.ConfigProperty> requiredService2Properties = new HashSet<Stack.ConfigProperty>();
+ requiredService2Properties.add(new Stack.ConfigProperty("category2", "prop2", null));
+
+
+ // Blueprint config
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ Map<String, String> hdfsProps = new HashMap<String, String>();
+ properties.put("hdfs-site", hdfsProps);
+ hdfsProps.put("foo", "val");
+ hdfsProps.put("bar", "val");
+ hdfsProps.put("secret", "SECRET:hdfs-site:1:test");
+
+ Map<String, String> category1Props = new HashMap<String, String>();
+ properties.put("category1", category1Props);
+ category1Props.put("prop1", "val");
+
+ Map<String, Map<String, Map<String, String>>> attributes = new HashMap<String, Map<String, Map<String, String>>>();
+ Configuration configuration = new Configuration(properties, attributes, EMPTY_CONFIGURATION);
+ // set config for group2 which contains a required property
+
+ replay(stack, group1, group2);
+
+ Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration);
+ try {
+ blueprint.validateRequiredProperties();
+ fail("Expected exception to be thrown for using secret reference");
+ } catch (InvalidTopologyException e) {
+ System.out.println("****" + e.getMessage() + "***");
+ }
+
+ }
+
//todo: ensure coverage for these existing tests
// private void validateEntity(BlueprintEntity entity, boolean containsConfig) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/11db1a42/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml
index 1c02e86..f762116 100644
--- a/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/test/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-site.xml
@@ -24,6 +24,12 @@
<!-- ResourceManager -->
<property>
+ <name>test.password</name>
+ <value> </value>
+ <property-type>PASSWORD</property-type>
+ </property>
+
+ <property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>localhost:8025</value>
<deleted>true</deleted>