You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2013/10/29 17:42:31 UTC
git commit: AMBARI-3600,
AMBARI-3601. Add stale_configs to host_component response and
config_types to stack service response (ncole)
Updated Branches:
refs/heads/trunk 1f26f6d3d -> 6115a5720
AMBARI-3600, AMBARI-3601. Add stale_configs to host_component response and config_types to stack service response (ncole)
Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/6115a572
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/6115a572
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/6115a572
Branch: refs/heads/trunk
Commit: 6115a5720ed206ba54fee233dd5d006c54267fde
Parents: 1f26f6d
Author: Nate Cole <nc...@hortonworks.com>
Authored: Mon Oct 28 11:56:39 2013 -0400
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Tue Oct 29 12:24:45 2013 -0400
----------------------------------------------------------------------
.../server/api/util/StackExtensionHelper.java | 6 +-
.../AmbariManagementControllerImpl.java | 138 +++-----
.../ServiceComponentHostResponse.java | 16 +
.../server/controller/StackServiceResponse.java | 12 +-
.../internal/HostComponentResourceProvider.java | 4 +
.../internal/StackServiceResourceProvider.java | 8 +-
.../ambari/server/state/ConfigHelper.java | 330 +++++++++++++++++++
.../apache/ambari/server/state/ServiceInfo.java | 82 ++++-
.../svccomphost/ServiceComponentHostImpl.java | 20 +-
.../src/main/resources/properties.json | 2 +
.../AmbariManagementControllerTest.java | 3 +-
.../svccomphost/ServiceComponentHostTest.java | 163 ++++++++-
12 files changed, 673 insertions(+), 111 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java
index 8feca20..6463150 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java
@@ -154,7 +154,7 @@ public class StackExtensionHelper {
List<ServiceInfo> serviceInfoList = parentStack.getServices();
for (ServiceInfo service : serviceInfoList) {
ServiceInfo existingService = serviceInfoMap.get(service.getName());
- if (service.isDeleted().booleanValue()) {
+ if (service.isDeleted()) {
serviceInfoMap.remove(service.getName());
continue;
}
@@ -185,6 +185,9 @@ public class StackExtensionHelper {
.FILENAME_FILTER);
if (servicesFolders != null) {
for (File serviceFolder : servicesFolders) {
+ if (!serviceFolder.isDirectory())
+ continue;
+
// Get information about service
ServiceInfo serviceInfo = new ServiceInfo();
serviceInfo.setName(serviceFolder.getName());
@@ -314,6 +317,7 @@ public class StackExtensionHelper {
pi.setFilename(propertyFile.getName());
list.add(pi);
}
+
return list;
} catch (Exception e) {
LOG.error("Could not load configuration for " + propertyFile, e);
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index cd7e957..78c3c90 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -18,11 +18,21 @@
package org.apache.ambari.server.controller;
-import com.google.gson.Gson;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-import com.google.inject.Singleton;
-import com.google.inject.persist.Transactional;
+import java.io.File;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.ClusterNotFoundException;
import org.apache.ambari.server.DuplicateResourceException;
@@ -55,6 +65,7 @@ import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.ComponentInfo;
import org.apache.ambari.server.state.Config;
import org.apache.ambari.server.state.ConfigFactory;
+import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.DesiredConfig;
import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.HostState;
@@ -87,20 +98,11 @@ import org.apache.http.client.utils.URIBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.text.MessageFormat;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeMap;
+import com.google.gson.Gson;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.Singleton;
+import com.google.inject.persist.Transactional;
@Singleton
public class AmbariManagementControllerImpl implements
@@ -1247,54 +1249,15 @@ public class AmbariManagementControllerImpl implements
}
}
- private void findConfigurationPropertiesWithOverrides(
+ private Map<String, Map<String, String>> findConfigurationPropertiesWithOverrides(
Map<String, Map<String, String>> configurations,
- Map<String, Map<String, String>> configTags,
Cluster cluster, String serviceName,
- Map<String, DesiredConfig> clusterDesiredConfigs,
Map<String, DesiredConfig> desiredConfigMap) throws AmbariException {
- // Do not use host component config mappings. Instead, the rules are:
- // 1) Use the cluster desired config
- // 2) override (1) with service-specific overrides
- // 3) override (2) with host-specific overrides
-
- for (Entry<String, DesiredConfig> entry : clusterDesiredConfigs.entrySet()) {
- String type = entry.getKey();
- String tag = entry.getValue().getVersion();
- // 1) start with cluster config
- Config config = cluster.getConfig(type, tag);
-
- if (null == config) {
- continue;
- }
-
- Map<String, String> props = new HashMap<String, String>(config.getProperties());
- Map<String, String> tags = new HashMap<String, String>();
- tags.put(CLUSTER_LEVEL_TAG, config.getVersionTag());
-
- // 2) apply the service overrides, if any are defined with different tags
- Service service = cluster.getService(serviceName);
- Config svcConfig = service.getDesiredConfigs().get(type);
- if (null != svcConfig && !svcConfig.getVersionTag().equals(tag)) {
- props.putAll(svcConfig.getProperties());
- //TODO why don't update tags with service overrides?
- tags.put("service_override_tag", svcConfig.getVersionTag());
- }
-
- // 3) apply the host overrides, if any
- DesiredConfig dc = desiredConfigMap.get(type);
-
- if (null != dc) {
- Config hostConfig = cluster.getConfig(type, dc.getVersion());
- if (null != hostConfig) {
- props.putAll(hostConfig.getProperties());
- tags.put("host_override_tag", hostConfig.getVersionTag());
- }
- }
-
- configTags.put(type, tags);
- }
+
+ ConfigHelper ch = injector.getInstance(ConfigHelper.class);
+
+ Map<String, Map<String,String>> configTags = ch.getEffectiveDesiredTags(cluster, serviceName, desiredConfigMap);
// HACK HACK HACK if the service has configs that are NOT included
@@ -1313,32 +1276,21 @@ public class AmbariManagementControllerImpl implements
configTags.put(type, tags);
}
}
+
+ return configTags;
}
- private void findConfigurationPropertiesWithOverrides(
+ private Map<String, Map<String,String>> findConfigurationPropertiesWithOverrides(
Map<String, Map<String, String>> configurations,
- Map<String, Map<String, String>> configTags,
- Cluster cluster, String serviceName, String hostName,
- Map<String, DesiredConfig> clusterDesiredConfigs) throws AmbariException {
+ Cluster cluster, String serviceName, String hostName) throws AmbariException {
Host host = clusters.getHost(hostName);
Map<String, DesiredConfig> desiredConfigMap = host.getDesiredConfigs(cluster.getClusterId());
- findConfigurationPropertiesWithOverrides(configurations, configTags, cluster,
- serviceName, clusterDesiredConfigs, desiredConfigMap);
+ return findConfigurationPropertiesWithOverrides(configurations, cluster,
+ serviceName, desiredConfigMap);
}
- private void findConfigurationPropertiesWithOverrides(
- Map<String, Map<String, String>> configurations,
- Map<String, Map<String, String>> configTags,
- Cluster cluster, String serviceName, String hostName) throws AmbariException {
-
- Map<String, DesiredConfig> clusterDesiredConfigs = cluster.getDesiredConfigs();
-
- findConfigurationPropertiesWithOverrides(configurations, configTags, cluster,
- serviceName, hostName, clusterDesiredConfigs);
-
- }
private List<Stage> doStageCreation(Cluster cluster,
Map<State, List<Service>> changedServices,
@@ -1348,7 +1300,6 @@ public class AmbariManagementControllerImpl implements
boolean runSmokeTest, boolean reconfigureClients)
throws AmbariException {
- Map<String, DesiredConfig> clusterDesiredConfigs = cluster.getDesiredConfigs();
// TODO handle different transitions?
// Say HDFS to stopped and MR to started, what order should actions be done
@@ -1531,10 +1482,8 @@ public class AmbariManagementControllerImpl implements
// [ type -> [ key, value ] ]
Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>();
- Map<String, Map<String, String>> configTags = new HashMap<String, Map<String, String>>();
-
- findConfigurationPropertiesWithOverrides(configurations, configTags, cluster, scHost.getServiceName(),
- clusterDesiredConfigs, configsByHosts.get(scHost.getHostName()));
+ Map<String, Map<String, String>> configTags = findConfigurationPropertiesWithOverrides(
+ configurations, cluster, scHost.getServiceName(), configsByHosts.get(scHost.getHostName()));
// HACK HACK HACK
if (!scHost.getHostName().equals(jobtrackerHost)) {
@@ -1577,10 +1526,8 @@ public class AmbariManagementControllerImpl implements
// [ type -> [ key, value ] ]
Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String,String>>();
- Map<String, Map<String, String>> configTags = new HashMap<String, Map<String,String>>();
-
- findConfigurationPropertiesWithOverrides(configurations, configTags,
- cluster, serviceName, clientHost, clusterDesiredConfigs);
+ Map<String, Map<String, String>> configTags = findConfigurationPropertiesWithOverrides(
+ configurations, cluster, serviceName, clientHost);
stage.getExecutionCommandWrapper(clientHost,
smokeTestRole).getExecutionCommand()
@@ -2739,10 +2686,7 @@ public class AmbariManagementControllerImpl implements
// [ type -> [ key, value ] ]
Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String,String>>();
- Map<String, Map<String, String>> configTags = new TreeMap<String,
- Map<String, String>>();
-
- findConfigurationPropertiesWithOverrides(configurations, configTags,
+ Map<String, Map<String, String>> configTags = findConfigurationPropertiesWithOverrides(configurations,
cluster, actionRequest.getServiceName(), hostName);
ExecutionCommand execCmd = stage.getExecutionCommandWrapper(hostName,
@@ -2793,11 +2737,11 @@ public class AmbariManagementControllerImpl implements
Map<String, Map<String, String>> configurations =
new TreeMap<String, Map<String, String>>();
- Map<String, Map<String, String>> configTags = new TreeMap<String,
- Map<String, String>>();
- findConfigurationPropertiesWithOverrides(configurations, configTags,
- cluster, serviceName, namenodeHost);
+
+ Map<String, Map<String, String>> configTags = findConfigurationPropertiesWithOverrides(
+ configurations, cluster, serviceName, namenodeHost);
+
// Add the tag for hdfs-exclude-file
Map<String, String> excludeTags = new HashMap<String, String>();
excludeTags.put(CLUSTER_LEVEL_TAG, config.getVersionTag());
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceComponentHostResponse.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceComponentHostResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceComponentHostResponse.java
index b3fdbee..eba7da8 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceComponentHostResponse.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceComponentHostResponse.java
@@ -50,6 +50,8 @@ public class ServiceComponentHostResponse {
private String ha_status = "NA";
+ private boolean staleConfig = false;
+
public ServiceComponentHostResponse(String clusterName, String serviceName,
String componentName, String hostname,
@@ -271,4 +273,18 @@ public class ServiceComponentHostResponse {
return actualConfigs;
}
+ /**
+ * @return if the configs are stale
+ */
+ public boolean isStaleConfig() {
+ return staleConfig;
+ }
+
+ /**
+ * @param stale
+ */
+ public void setStaleConfig(boolean stale) {
+ staleConfig = stale;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java
index d9cbf6e..4aa3f49 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/StackServiceResponse.java
@@ -18,6 +18,8 @@
package org.apache.ambari.server.controller;
+import java.util.List;
+
public class StackServiceResponse {
private String serviceName;
@@ -27,12 +29,16 @@ public class StackServiceResponse {
private String comments;
private String serviceVersion;
+
+ private List<String> configTypes;
- public StackServiceResponse(String serviceName, String userName, String comments, String serviceVersion) {
+ public StackServiceResponse(String serviceName, String userName, String comments, String serviceVersion,
+ List<String> types) {
setServiceName(serviceName);
setUserName(userName);
setComments(comments);
setServiceVersion(serviceVersion);
+ configTypes = types;
}
public String getServiceName() {
@@ -66,5 +72,9 @@ public class StackServiceResponse {
public void setServiceVersion(String serviceVersion) {
this.serviceVersion = serviceVersion;
}
+
+ public List<String> getConfigTypes() {
+ return configTypes;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java
index f9dd801..006e59b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java
@@ -66,6 +66,8 @@ class HostComponentResourceProvider extends AbstractControllerResourceProvider {
= PropertyHelper.getPropertyId("HostRoles", "desired_stack_id");
protected static final String HOST_COMPONENT_ACTUAL_CONFIGS_PROPERTY_ID
= PropertyHelper.getPropertyId("HostRoles", "actual_configs");
+ protected static final String HOST_COMPONENT_STALE_CONFIGS_PROPERTY_ID
+ = PropertyHelper.getPropertyId("HostRoles", "stale_configs");
//Component name mappings
private static final Map<String, PropertyProvider> HOST_COMPONENT_PROPERTIES_PROVIDER = new HashMap<String, PropertyProvider>();
@@ -186,6 +188,8 @@ class HostComponentResourceProvider extends AbstractControllerResourceProvider {
response.getDesiredConfigs(), requestedIds);
setResourceProperty(resource, HOST_COMPONENT_ACTUAL_CONFIGS_PROPERTY_ID,
response.getActualConfigs(), requestedIds);
+ setResourceProperty(resource, HOST_COMPONENT_STALE_CONFIGS_PROPERTY_ID,
+ Boolean.valueOf(response.isStaleConfig()), requestedIds);
String componentName = (String)resource.getPropertyValue(HOST_COMPONENT_COMPONENT_NAME_PROPERTY_ID);
PropertyProvider propertyProvider = HOST_COMPONENT_PROPERTIES_PROVIDER.get(componentName);
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackServiceResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackServiceResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackServiceResourceProvider.java
index 1c45537..3265782 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackServiceResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackServiceResourceProvider.java
@@ -58,7 +58,10 @@ public class StackServiceResourceProvider extends ReadOnlyResourceProvider {
.getPropertyId("StackServices", "comments");
private static final String VERSION_PROPERTY_ID = PropertyHelper
- .getPropertyId("StackServices", "service_version");;
+ .getPropertyId("StackServices", "service_version");
+
+ private static final String CONFIG_TYPES = PropertyHelper
+ .getPropertyId("StackServices", "config_types");
private static Set<String> pkPropertyIds = new HashSet<String>(
Arrays.asList(new String[] { STACK_NAME_PROPERTY_ID,
@@ -109,6 +112,9 @@ public class StackServiceResourceProvider extends ReadOnlyResourceProvider {
setResourceProperty(resource, VERSION_PROPERTY_ID,
response.getServiceVersion(), requestedIds);
+
+ setResourceProperty(resource, CONFIG_TYPES,
+ response.getConfigTypes(), requestedIds);
resources.add(resource);
}
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
new file mode 100644
index 0000000..855711b
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
@@ -0,0 +1,330 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.state;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
+
+import com.google.inject.Inject;
+
+/**
+ * Helper class that works with config traversals.
+ */
+public class ConfigHelper {
+
+ private Clusters clusters = null;
+ private AmbariMetaInfo ambariMetaInfo = null;
+
+ @Inject
+ public ConfigHelper(Clusters c, AmbariMetaInfo metaInfo) {
+ clusters = c;
+ ambariMetaInfo = metaInfo;
+ }
+
+ /**
+ * Gets the desired tags for a cluster and host
+ * @param cluster the cluster
+ * @param serviceName the optional service name
+ * @param hostName the host name
+ * @return a map of tag type to tag names with overrides
+ * @throws AmbariException
+ */
+ public Map<String, Map<String, String>> getEffectiveDesiredTags(
+ Cluster cluster, String serviceName, String hostName) throws AmbariException {
+
+ Host host = clusters.getHost(hostName);
+ Map<String, DesiredConfig> hostDesired = host.getDesiredConfigs(cluster.getClusterId());
+
+ return getEffectiveDesiredTags(cluster, serviceName, hostDesired);
+ }
+
+ /**
+ * Gets the desired tags for a cluster and host
+ * @param cluster the cluster
+ * @param serviceName the optional service name
+ * @param hostDesired the optional host desired configs
+ * @return a map of tag type to tag names with overrides
+ * @throws AmbariException
+ */
+ public Map<String, Map<String, String>> getEffectiveDesiredTags(
+ Cluster cluster, String serviceName, Map<String, DesiredConfig> hostDesired) throws AmbariException {
+
+ Map<String, DesiredConfig> clusterDesired = cluster.getDesiredConfigs();
+
+ Map<String, Map<String,String>> resolved = new TreeMap<String, Map<String, String>>();
+
+ // Do not use host component config mappings. Instead, the rules are:
+ // 1) Use the cluster desired config
+ // 2) override (1) with service-specific overrides
+ // 3) override (2) with host-specific overrides
+
+ for (Entry<String, DesiredConfig> clusterEntry : clusterDesired.entrySet()) {
+ String type = clusterEntry.getKey();
+ String tag = clusterEntry.getValue().getVersion();
+
+ // 1) start with cluster config
+ Config config = cluster.getConfig(type, tag);
+ if (null == config) {
+ continue;
+ }
+
+ Map<String, String> tags = new LinkedHashMap<String, String>();
+
+ tags.put("tag", config.getVersionTag());
+
+ // 2) apply the service overrides, if any are defined with different tags
+ if (null != serviceName) {
+ Service service = cluster.getService(serviceName);
+ Config svcConfig = service.getDesiredConfigs().get(type);
+ if (null != svcConfig && !svcConfig.getVersionTag().equals(tag)) {
+ tags.put("service_override_tag", svcConfig.getVersionTag());
+ }
+ }
+
+ if (null != hostDesired) {
+ // 3) apply the host overrides, if any
+ DesiredConfig dc = hostDesired.get(type);
+
+ if (null != dc) {
+ Config hostConfig = cluster.getConfig(type, dc.getVersion());
+ if (null != hostConfig) {
+ tags.put("host_override_tag", hostConfig.getVersionTag());
+ }
+ }
+ }
+
+ resolved.put(type, tags);
+ }
+
+ return resolved;
+ }
+
+ /**
+ * The purpose of this method is to determine if a {@link ServiceComponentHost}'s
+ * known actual configs are different than what is set on the cluster (the desired).
+ * The following logic is applied:
+ * <ul>
+ * <li>Desired type does not exist on the SCH (actual)
+ * <ul>
+ * <li>Type does not exist on the stack: <code>false</code></li>
+ * <li>Type exists on the stack: <code>true</code> if the config key is on the stack.
+ * otherwise <code>false</code></li>
+ * </ul>
+ * </li>
+ * <li> Desired type exists for the SCH
+ * <ul>
+ * <li>Desired tags already set for the SCH (actual): <code>false</code></li>
+ * <li>Desired tags DO NOT match SCH: <code>true</code> if the changed keys
+ * exist on the stack, otherwise <code>false</code></li>
+ * </ul>
+ * </li>
+ * </ul>
+ * @param serviceComponentHostImpl
+ * @return <code>true</code> if the actual configs are stale
+ */
+ public boolean isStaleConfigs(ServiceComponentHost sch) throws AmbariException {
+
+ Map<String, DesiredConfig> actual = sch.getActualConfigs();
+ if (null == actual || actual.isEmpty())
+ return false;
+
+ Cluster cluster = clusters.getClusterById(sch.getClusterId());
+ StackId stackId = cluster.getDesiredStackVersion();
+
+ Map<String, Map<String, String>> desired = getEffectiveDesiredTags(cluster,
+ sch.getServiceName(), sch.getHostName());
+
+ ServiceInfo serviceInfo = ambariMetaInfo.getService(stackId.getStackName(),
+ stackId.getStackVersion(), sch.getServiceName());
+
+ // Configs are considered stale when:
+ // - desired type DOES NOT exist in actual
+ // --- desired type DOES NOT exist in stack: not_stale
+ // --- desired type DOES exist in stack: check stack for any key: stale
+ // - desired type DOES exist in actual
+ // --- desired tags DO match actual tags: not_stale
+ // --- desired tags DO NOT match actual tags
+ // ---- merge values, determine changed keys, check stack: stale
+
+ boolean stale = false;
+
+ Iterator<Entry<String, Map<String, String>>> it = desired.entrySet().iterator();
+
+ while (it.hasNext() && !stale) {
+ Entry<String, Map<String, String>> desiredEntry = it.next();
+
+ String type = desiredEntry.getKey();
+ Map<String, String> tags = desiredEntry.getValue();
+
+ if (!actual.containsKey(type)) {
+ // desired is set, but actual is not
+ if (!serviceInfo.hasConfigType(type)) {
+ stale = false;
+ } else {
+ // find out if the keys are stale by first checking the target service,
+ // then all services
+ Collection<String> keys = mergeKeyNames(cluster, type, tags.values());
+
+ if (serviceInfo.hasPropertyFor(type, keys) || !hasPropertyFor(stackId, type, keys))
+ stale = true;
+ }
+ } else {
+ // desired and actual both define the type
+ DesiredConfig dc = actual.get(type);
+ Map<String, String> actualTags = buildTags(dc);
+
+ if (!isTagChange(tags, actualTags)) {
+ stale = false;
+ } else {
+ // tags are change, need to find out what has changed, and if it applies
+ // to the service
+ Collection<String> changed = findChangedKeys(cluster, type, tags.values(), actualTags.values());
+ if (serviceInfo.hasPropertyFor(type, changed)) {
+ stale = true;
+ }
+ }
+
+ }
+ }
+ return stale;
+ }
+
+ /**
+ * @return <code>true</code> if any service on the stack defines a property
+ * for the type.
+ */
+ private boolean hasPropertyFor(StackId stack, String type,
+ Collection<String> keys) throws AmbariException {
+
+ for (ServiceInfo svc : ambariMetaInfo.getServices(stack.getStackName(),
+ stack.getStackVersion()).values()) {
+
+ if (svc.hasPropertyFor(type, keys))
+ return true;
+
+ }
+
+ return false;
+ }
+
+ /**
+ * @return the keys that have changed values
+ */
+ private Collection<String> findChangedKeys(Cluster cluster, String type,
+ Collection<String> desiredTags, Collection<String> actualTags) {
+
+ Map<String, String> desiredValues = new HashMap<String, String>();
+ Map<String, String> actualValues = new HashMap<String, String>();
+
+ for (String tag : desiredTags) {
+ Config config = cluster.getConfig(type, tag);
+ if (null != config)
+ desiredValues.putAll(config.getProperties());
+ }
+
+ for (String tag : actualTags) {
+ Config config = cluster.getConfig(type, tag);
+ if (null != config)
+ actualValues.putAll(config.getProperties());
+ }
+
+ List<String> keys = new ArrayList<String>();
+
+ for (Entry<String, String> entry : desiredValues.entrySet()) {
+ String key = entry.getKey();
+ String value = entry.getValue();
+
+ if (!actualValues.containsKey(key))
+ keys.add(key);
+ else if (!actualValues.get(key).equals(value))
+ keys.add(key);
+ }
+
+ return keys;
+ }
+
+ /**
+ * @return the map of tags for a desired config
+ */
+ private Map<String, String> buildTags(DesiredConfig dc) {
+ Map<String, String> map = new LinkedHashMap<String, String>();
+ map.put("tag", dc.getVersion());
+ if (null != dc.getServiceName())
+ map.put("service_override_tag", dc.getServiceName());
+ if (0 != dc.getHostOverrides().size())
+ map.put("host_override_tag", dc.getHostOverrides().get(0).getVersionTag());
+
+ return map;
+ }
+
+ /**
+ * @return true if the tags are different in any way, even if not-specified
+ */
+ private boolean isTagChange(Map<String, String> desiredTags, Map<String, String> actualTags) {
+ if (!actualTags.get("tag").equals (desiredTags.get("tag")))
+ return true;
+
+ String tag0 = actualTags.get("service_override_tag");
+ String tag1 = desiredTags.get("service_override_tag");
+ tag0 = (null == tag0) ? "" : tag0;
+ tag1 = (null == tag1) ? "" : tag1;
+ if (!tag0.equals(tag1))
+ return true;
+
+ // desired config can only have one value here since it's from the HC.
+ tag0 = actualTags.get("host_override_tag");
+ tag1 = desiredTags.get("host_override_tag");
+ tag0 = (null == tag0) ? "" : tag0;
+ tag1 = (null == tag1) ? "" : tag1;
+ if (!tag0.equals(tag1))
+ return true;
+
+ return false;
+ }
+
+ /**
+ * @return the list of combined config property names
+ */
+ private Collection<String> mergeKeyNames(Cluster cluster, String type, Collection<String> tags) {
+ Set<String> names = new HashSet<String>();
+
+ for (String tag : tags) {
+ Config config = cluster.getConfig(type, tag);
+ if (null != config) {
+ names.addAll(config.getProperties().keySet());
+ }
+ }
+
+ return names;
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java
index 937d1de..fa2c759 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java
@@ -19,9 +19,15 @@
package org.apache.ambari.server.state;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.ambari.server.controller.StackServiceResponse;
+import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.map.annotate.JsonFilter;
@JsonFilter("propertiesfilter")
@@ -32,14 +38,16 @@ public class ServiceInfo {
private String comment;
private List<PropertyInfo> properties;
private List<ComponentInfo> components;
- private Boolean isDeleted = false;
+ private boolean isDeleted = false;
+ @JsonIgnore
+ private volatile Map<String, Set<String>> configLayout = null;
- public Boolean isDeleted() {
+ public boolean isDeleted() {
return isDeleted;
}
public void setDeleted(boolean deleted) {
- isDeleted = Boolean.valueOf(deleted);
+ isDeleted = deleted;
}
public String getName() {
@@ -128,6 +136,72 @@ public class ServiceInfo {
public StackServiceResponse convertToResponse()
{
- return new StackServiceResponse(getName(), getUser(), getComment(), getVersion());
+ return new StackServiceResponse(getName(), getUser(), getComment(), getVersion(),
+ getConfigTypes());
+ }
+
+ public List<String> getConfigTypes() {
+ buildConfigLayout();
+ return new ArrayList<String>(configLayout.keySet());
+ }
+
+
+ /**
+ * @param type the config type
+ * @return <code>true</code> if the service defines the supplied type
+ */
+ public boolean hasConfigType(String type) {
+ buildConfigLayout();
+
+ return configLayout.containsKey(type);
+ }
+
+ /**
+ * The purpose of this method is to determine if a service has a property
+ * defined in a supplied set:
+ * <ul>
+ * <li>If the type is not defined for the service, then no property can exist.</li>
+ * <li>If the type is defined, then check each supplied property for existence.</li>
+ * </ul>
+ * @param type the config type
+ * @param keyNames the names of all the config keys for the given type
+ * @return <code>true</code> if the config is stale
+ */
+ public boolean hasPropertyFor(String type, Collection<String> keyNames) {
+ if (!hasConfigType(type))
+ return false;
+
+ Set<String> keys = configLayout.get(type);
+
+ for (String staleCheck : keyNames) {
+ if (keys.contains(staleCheck))
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * Builds the config map specific to this service.
+ */
+ private void buildConfigLayout() {
+ if (null == configLayout) {
+ synchronized(this) {
+ if (null == configLayout) {
+ configLayout = new HashMap<String, Set<String>>();
+
+ for (PropertyInfo pi : getProperties()) {
+ String type = pi.getFilename();
+ int idx = type.indexOf(".xml");
+ type = type.substring(0, idx);
+
+ if (!configLayout.containsKey(type))
+ configLayout.put(type, new HashSet<String>());
+
+ configLayout.get(type).add(pi.getName());
+ }
+ }
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
index 1e49808..2c79f96 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
@@ -87,11 +87,12 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
@Inject
Clusters clusters;
@Inject
- HostComponentDesiredConfigMappingDAO
- hostComponentDesiredConfigMappingDAO;
+ HostComponentDesiredConfigMappingDAO hostComponentDesiredConfigMappingDAO;
@Inject
- HostComponentConfigMappingDAO
- hostComponentConfigMappingDAO;
+ HostComponentConfigMappingDAO hostComponentConfigMappingDAO;
+
+ @Inject
+ ConfigHelper helper;
private HostComponentStateEntity stateEntity;
private HostComponentDesiredStateEntity desiredStateEntity;
@@ -1316,6 +1317,13 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
r.setHa_status(ha_status);
r.setActualConfigs(actualConfigs);
+
+ try {
+ r.setStaleConfig(helper.isStaleConfigs(this));
+ } catch (Exception e) {
+ LOG.error("Could not determine stale config", e);
+ }
+
return r;
} finally {
readLock.unlock();
@@ -1559,6 +1567,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
try {
writeLock.lock();
try {
+
actualConfigs = new HashMap<String, DesiredConfig>();
String hostName = getHostName();
@@ -1571,6 +1580,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
String hostTag = values.get("host_override_tag");
DesiredConfig dc = new DesiredConfig();
+ dc.setServiceName(values.get("service_override_tag"));
dc.setVersion(tag);
actualConfigs.put(type, dc);
if (null != hostTag && null != hostName) {
@@ -1619,4 +1629,6 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
}
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/main/resources/properties.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/properties.json b/ambari-server/src/main/resources/properties.json
index 00abfac..2a985c7 100644
--- a/ambari-server/src/main/resources/properties.json
+++ b/ambari-server/src/main/resources/properties.json
@@ -68,6 +68,7 @@
"params/run_smoke_test",
"HostRoles/nagios_alerts",
"HostRoles/ha_status",
+ "HostRoles/stale_configs",
"_"
],
"Configuration":[
@@ -156,6 +157,7 @@
"StackServices/user_name",
"StackServices/comments",
"StackServices/service_version",
+ "StackServices/config_types",
"_"
],
"StackConfiguration":[
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index 2206062..1e8551a 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -5794,8 +5794,9 @@ public class AmbariManagementControllerTest {
Assert.assertEquals(1, responsesWithParams.size());
for (StackServiceResponse responseWithParams: responsesWithParams) {
Assert.assertEquals(responseWithParams.getServiceName(), SERVICE_NAME);
-
+ Assert.assertTrue(responseWithParams.getConfigTypes().size() > 0);
}
+
StackServiceRequest invalidRequest = new StackServiceRequest(STACK_NAME, STACK_VERSION, NON_EXT_VALUE);
try {
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/6115a572/ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java
index b47c64f..050defa 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java
@@ -99,12 +99,21 @@ public class ServiceComponentHostTest {
public void teardown() {
injector.getInstance(PersistService.class).stop();
}
-
+
private ServiceComponentHost createNewServiceComponentHost(
String svc,
String svcComponent,
String hostName, boolean isClient) throws AmbariException{
Cluster c = clusters.getCluster("C1");
+
+ return createNewServiceComponentHost(c, svc, svcComponent, hostName, isClient);
+ }
+ private ServiceComponentHost createNewServiceComponentHost(
+ Cluster c,
+ String svc,
+ String svcComponent,
+ String hostName, boolean isClient) throws AmbariException{
+
Service s = null;
try {
@@ -133,7 +142,7 @@ public class ServiceComponentHostTest {
impl.getState());
Assert.assertEquals(State.INIT,
impl.getDesiredState());
- Assert.assertEquals("C1", impl.getClusterName());
+ Assert.assertEquals(c.getClusterName(), impl.getClusterName());
Assert.assertEquals(c.getClusterId(), impl.getClusterId());
Assert.assertEquals(s.getName(), impl.getServiceName());
Assert.assertEquals(sc.getName(), impl.getServiceComponentName());
@@ -525,6 +534,8 @@ public class ServiceComponentHostTest {
Assert.assertEquals(State.INSTALLED.toString(), r.getDesiredState());
Assert.assertEquals(State.INSTALLING.toString(), r.getLiveState());
Assert.assertEquals("HDP-1.0.0", r.getStackVersion());
+
+ Assert.assertFalse(r.isStaleConfig());
// TODO check configs
@@ -713,4 +724,152 @@ public class ServiceComponentHostTest {
}
}
}
+
+ @Test
+ public void testStaleConfigs() throws Exception {
+ String stackVersion="HDP-2.0.6";
+ String clusterName = "c2";
+ String hostName = "h3";
+
+ clusters.addCluster(clusterName);
+ clusters.addHost(hostName);
+ clusters.getHost(hostName).setOsType("centos5");
+ clusters.getHost(hostName).persist();
+ clusters.getCluster(clusterName).setDesiredStackVersion(
+ new StackId(stackVersion));
+ metaInfo.init();
+ clusters.mapHostToCluster(hostName, clusterName);
+
+ Cluster cluster = clusters.getCluster(clusterName);
+
+ ServiceComponentHost sch1 = createNewServiceComponentHost(cluster, "HDFS", "NAMENODE", hostName, false);
+ ServiceComponentHost sch2 = createNewServiceComponentHost(cluster, "HDFS", "DATANODE", hostName, false);
+ ServiceComponentHost sch3 = createNewServiceComponentHost(cluster, "MAPREDUCE2", "HISTORYSERVER", hostName, false);
+
+ sch1.setDesiredState(State.INSTALLED);
+ sch1.setState(State.INSTALLING);
+ sch1.setStackVersion(new StackId(stackVersion));
+
+ sch2.setDesiredState(State.INSTALLED);
+ sch2.setState(State.INSTALLING);
+ sch2.setStackVersion(new StackId(stackVersion));
+
+ sch3.setDesiredState(State.INSTALLED);
+ sch3.setState(State.INSTALLING);
+ sch3.setStackVersion(new StackId(stackVersion));
+
+ Assert.assertFalse(sch1.convertToResponse().isStaleConfig());
+ Assert.assertFalse(sch2.convertToResponse().isStaleConfig());
+
+ makeConfig(cluster, "global", "version1",
+ new HashMap<String,String>() {{
+ put("a", "b");
+ put("dfs_namenode_name_dir", "/foo1"); // HDFS only
+ put("mapred_log_dir_prefix", "/foo2"); // MR2 only
+ }});
+
+ Map<String, Map<String, String>> actual = new HashMap<String, Map<String, String>>() {{
+ put("global", new HashMap<String,String>() {{ put("tag", "version1"); }});
+ }};
+
+ sch1.updateActualConfigs(actual);
+ sch2.updateActualConfigs(actual);
+ sch3.updateActualConfigs(actual);
+
+ makeConfig(cluster, "foo", "version1",
+ new HashMap<String,String>() {{ put("a", "c"); }});
+
+ // HDP-x/HDFS does not define type 'foo', so changes do not count to stale
+ Assert.assertFalse(sch1.convertToResponse().isStaleConfig());
+ Assert.assertFalse(sch2.convertToResponse().isStaleConfig());
+
+ makeConfig(cluster, "hdfs-site", "version1",
+ new HashMap<String,String>() {{ put("a", "b"); }});
+
+ // HDP-x/HDFS/hdfs-site is not on the actual, but it is defined, so it is stale
+ Assert.assertTrue(sch1.convertToResponse().isStaleConfig());
+ Assert.assertTrue(sch2.convertToResponse().isStaleConfig());
+
+ actual.put("hdfs-site", new HashMap<String, String>() {{ put ("tag", "version1"); }});
+
+ sch1.updateActualConfigs(actual);
+ // HDP-x/HDFS/hdfs-site up to date, only for sch1
+ Assert.assertFalse(sch1.convertToResponse().isStaleConfig());
+ Assert.assertTrue(sch2.convertToResponse().isStaleConfig());
+
+ sch2.updateActualConfigs(actual);
+ // HDP-x/HDFS/hdfs-site up to date for both
+ Assert.assertFalse(sch1.convertToResponse().isStaleConfig());
+ Assert.assertFalse(sch2.convertToResponse().isStaleConfig());
+
+ makeConfig(cluster, "hdfs-site", "version2",
+ new HashMap<String, String>() {{ put("dfs.journalnode.http-address", "http://foo"); }});
+
+ // HDP-x/HDFS/hdfs-site updated to changed property
+ Assert.assertTrue(sch1.convertToResponse().isStaleConfig());
+ Assert.assertTrue(sch2.convertToResponse().isStaleConfig());
+
+ actual.get("hdfs-site").put("tag", "version2");
+ sch1.updateActualConfigs(actual);
+ sch2.updateActualConfigs(actual);
+ // HDP-x/HDFS/hdfs-site updated to changed property
+ Assert.assertFalse(sch1.convertToResponse().isStaleConfig());
+ Assert.assertFalse(sch2.convertToResponse().isStaleConfig());
+
+ // make a host override
+ Host host = clusters.getHostsForCluster(cluster.getClusterName()).get(hostName);
+ Assert.assertNotNull(host);
+
+ Config c = configFactory.createNew(cluster, "hdfs-site",
+ new HashMap<String, String>() {{ put("dfs.journalnode.http-address", "http://goo"); }});
+ c.setVersionTag("version3");
+ c.persist();
+ cluster.addConfig(c);
+ host.addDesiredConfig(cluster.getClusterId(), true, "user", c);
+
+ // HDP-x/HDFS/hdfs-site updated host to changed property
+ Assert.assertTrue(sch1.convertToResponse().isStaleConfig());
+ Assert.assertTrue(sch2.convertToResponse().isStaleConfig());
+
+ actual.get("hdfs-site").put("host_override_tag", "version3");
+ sch2.updateActualConfigs(actual);
+ // HDP-x/HDFS/hdfs-site updated host to changed property
+ Assert.assertTrue(sch1.convertToResponse().isStaleConfig());
+ Assert.assertFalse(sch2.convertToResponse().isStaleConfig());
+
+ sch1.updateActualConfigs(actual);
+ // HDP-x/HDFS/hdfs-site updated host to changed property
+ Assert.assertFalse(sch1.convertToResponse().isStaleConfig());
+ Assert.assertFalse(sch2.convertToResponse().isStaleConfig());
+
+ // change 'global' property only affecting global/HDFS
+ makeConfig(cluster, "global", "version2",
+ new HashMap<String,String>() {{
+ put("a", "b");
+ put("dfs_namenode_name_dir", "/foo3"); // HDFS only
+ put("mapred_log_dir_prefix", "/foo2"); // MR2 only
+ }});
+
+ Assert.assertTrue(sch1.convertToResponse().isStaleConfig());
+ Assert.assertTrue(sch2.convertToResponse().isStaleConfig());
+ Assert.assertFalse(sch3.convertToResponse().isStaleConfig());
+ }
+
+ /**
+ * Helper method to create a configuration
+ * @param cluster the cluster
+ * @param type the config type
+ * @param tag the config tag
+ * @param values the values for the config
+ */
+ private void makeConfig(Cluster cluster, String type, String tag, Map<String, String> values) {
+ Config config = configFactory.createNew(cluster, type, values);
+ config.setVersionTag(tag);
+ config.persist();
+ cluster.addConfig(config);
+ cluster.addDesiredConfig("user", config);
+ }
+
+
+
}