You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2016/04/21 15:28:08 UTC
[2/3] ambari git commit: AMBARI-15994. Restart button took about 5
minutes to appear after a config change for zookeeper (dlysnichenko)
AMBARI-15994. Restart button took about 5 minutes to appear after a config change for zookeeper (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2bc94a04
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2bc94a04
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2bc94a04
Branch: refs/heads/branch-2.2
Commit: 2bc94a04b1603aa6b2be7a9ffe7e661c01d996cf
Parents: 4e8670e
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Thu Apr 21 16:26:48 2016 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Thu Apr 21 16:26:48 2016 +0300
----------------------------------------------------------------------
.../server/configuration/Configuration.java | 10 ++
.../ambari/server/state/ConfigHelper.java | 30 +++--
.../org/apache/ambari/server/state/Host.java | 9 ++
.../ambari/server/state/host/HostImpl.java | 21 +++-
.../ambari/server/state/ConfigHelperTest.java | 125 +++++++++++++++++++
5 files changed, 184 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2bc94a04/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index 0446469..6bcf539 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -185,6 +185,7 @@ public class Configuration {
public static final String SERVER_HRC_STATUS_SUMMARY_CACHE_SIZE = "server.hrcStatusSummary.cache.size";
public static final String SERVER_HRC_STATUS_SUMMARY_CACHE_EXPIRY_DURATION = "server.hrcStatusSummary.cache.expiryDuration";
public static final String SERVER_STALE_CONFIG_CACHE_ENABLED_KEY = "server.cache.isStale.enabled";
+ public static final String SERVER_STALE_CONFIG_CACHE_EXPIRATION_KEY = "server.cache.isStale.expiration";
public static final String SERVER_PERSISTENCE_TYPE_KEY = "server.persistence.type";
public static final String SERVER_JDBC_USER_NAME_KEY = "server.jdbc.user.name";
public static final String SERVER_JDBC_USER_PASSWD_KEY = "server.jdbc.user.passwd";
@@ -382,6 +383,7 @@ public class Configuration {
private static final long SERVER_EC_CACHE_SIZE_DEFAULT = 10000L;
private static final String SERVER_STALE_CONFIG_CACHE_ENABLED_DEFAULT = "true";
+ private static final String SERVER_STALE_CONFIG_CACHE_EXPIRATION_DEFAULT = "60";
private static final String SERVER_JDBC_USER_NAME_DEFAULT = "ambari";
private static final String SERVER_JDBC_USER_PASSWD_DEFAULT = "bigdata";
private static final String SERVER_JDBC_RCA_USER_NAME_DEFAULT = "mapred";
@@ -1875,6 +1877,14 @@ public class Configuration {
}
/**
+ * @return expiration time of stale config cache
+ */
+ public Integer staleConfigCacheExpiration() {
+ return Integer.parseInt(properties.getProperty(SERVER_STALE_CONFIG_CACHE_EXPIRATION_KEY,
+ SERVER_STALE_CONFIG_CACHE_EXPIRATION_DEFAULT));
+ }
+
+ /**
* @return a string array of suffixes used to validate repo URLs.
*/
public String[] getRepoValidationSuffixes(String osType) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/2bc94a04/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
index b21cbbd..7ddaee4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
@@ -62,7 +62,7 @@ public class ConfigHelper {
private static final String DELETED = "DELETED_";
public static final String CLUSTER_DEFAULT_TAG = "tag";
private final boolean STALE_CONFIGS_CACHE_ENABLED;
- private final int STALE_CONFIGS_CACHE_EXPIRATION_TIME = 300;
+ private final int STALE_CONFIGS_CACHE_EXPIRATION_TIME;
private final Cache<ServiceComponentHost, Boolean> staleConfigsCache;
private static final Logger LOG =
@@ -89,6 +89,7 @@ public class ConfigHelper {
ambariMetaInfo = metaInfo;
this.clusterDAO = clusterDAO;
STALE_CONFIGS_CACHE_ENABLED = configuration.isStaleConfigCacheEnabled();
+ STALE_CONFIGS_CACHE_EXPIRATION_TIME = configuration.staleConfigCacheExpiration();
staleConfigsCache = CacheBuilder.newBuilder().
expireAfterWrite(STALE_CONFIGS_CACHE_EXPIRATION_TIME, TimeUnit.SECONDS).build();
}
@@ -104,9 +105,24 @@ public class ConfigHelper {
public Map<String, Map<String, String>> getEffectiveDesiredTags(
Cluster cluster, String hostName) throws AmbariException {
+ return getEffectiveDesiredTags(cluster, hostName, false);
+ }
+
+ /**
+ * Gets the desired tags for a cluster and host
+ *
+ * @param cluster the cluster
+ * @param hostName the host name
+ * @param bypassCache don't use cached values
+ * @return a map of tag type to tag names with overrides
+ * @throws AmbariException
+ */
+ public Map<String, Map<String, String>> getEffectiveDesiredTags(
+ Cluster cluster, String hostName, boolean bypassCache) throws AmbariException {
+
Host host = (hostName == null) ? null : clusters.getHost(hostName);
- Map<String, HostConfig> desiredHostConfigs = (host == null) ? null : host.getDesiredHostConfigs(cluster);
- return getEffectiveDesiredTags(cluster, desiredHostConfigs);
+ Map<String, HostConfig> desiredHostConfigs = (host == null) ? null : host.getDesiredHostConfigs(cluster, bypassCache);
+ return getEffectiveDesiredTags(cluster, desiredHostConfigs, bypassCache);
}
/**
@@ -114,12 +130,13 @@ public class ConfigHelper {
*
* @param cluster the cluster
* @param hostConfigOverrides the host overrides applied using config groups
+ * @param bypassCache don't use cached values
* @return a map of tag type to tag names with overrides
*/
private Map<String, Map<String, String>> getEffectiveDesiredTags(
- Cluster cluster, Map<String, HostConfig> hostConfigOverrides) {
+ Cluster cluster, Map<String, HostConfig> hostConfigOverrides, boolean bypassCache) {
- Map<String, DesiredConfig> clusterDesired = (cluster == null) ? new HashMap<String, DesiredConfig>() : cluster.getDesiredConfigs(true);
+ Map<String, DesiredConfig> clusterDesired = (cluster == null) ? new HashMap<String, DesiredConfig>() : cluster.getDesiredConfigs(bypassCache);
Map<String, Map<String, String>> resolved = new TreeMap<String, Map<String, String>>();
@@ -981,8 +998,7 @@ public class ConfigHelper {
Cluster cluster = clusters.getClusterById(sch.getClusterId());
StackId stackId = cluster.getDesiredStackVersion();
- Map<String, Map<String, String>> desired = getEffectiveDesiredTags(cluster,
- sch.getHostName());
+ Map<String, Map<String, String>> desired = getEffectiveDesiredTags(cluster, sch.getHostName(), true);
ServiceInfo serviceInfo = ambariMetaInfo.getService(stackId.getStackName(),
stackId.getStackVersion(), sch.getServiceName());
http://git-wip-us.apache.org/repos/asf/ambari/blob/2bc94a04/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java b/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java
index 34fb995..ca854db 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Host.java
@@ -373,6 +373,15 @@ public interface Host {
Map<String, HostConfig> getDesiredHostConfigs(Cluster cluster) throws AmbariException;
/**
+ * Get the desired configurations for the host including overrides
+ * @param cluster
+ * @param bypassCache
+ * @return
+ * @throws AmbariException
+ */
+ Map<String, HostConfig> getDesiredHostConfigs(Cluster cluster, boolean bypassCache) throws AmbariException;
+
+ /**
* Sets the maintenance state for the host.
* @param clusterId the cluster id
* @param state the state
http://git-wip-us.apache.org/repos/asf/ambari/blob/2bc94a04/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
index 4b747fa..d221112 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java
@@ -1327,12 +1327,25 @@ public class HostImpl implements Host {
*/
@Override
public Map<String, HostConfig> getDesiredHostConfigs(Cluster cluster) throws AmbariException {
+ return getDesiredHostConfigs(cluster, false);
+ }
+
+ /**
+ * Get a map of configType with all applicable config tags.
+ *
+ * @param cluster the cluster
+ * @param bypassCache don't use cached values
+ *
+ * @return Map of configType -> HostConfig
+ */
+ @Override
+ public Map<String, HostConfig> getDesiredHostConfigs(Cluster cluster, boolean bypassCache) throws AmbariException {
Map<String, HostConfig> hostConfigMap = new HashMap<String, HostConfig>();
- Map<String, DesiredConfig> clusterDesiredConfigs = (cluster == null) ? new HashMap<String, DesiredConfig>() : cluster.getDesiredConfigs();
+ Map<String, DesiredConfig> clusterDesiredConfigs = (cluster == null) ? new HashMap<String, DesiredConfig>() : cluster.getDesiredConfigs(bypassCache);
if (clusterDesiredConfigs != null) {
for (Map.Entry<String, DesiredConfig> desiredConfigEntry
- : clusterDesiredConfigs.entrySet()) {
+ : clusterDesiredConfigs.entrySet()) {
HostConfig hostConfig = new HostConfig();
hostConfig.setDefaultVersionTag(desiredConfigEntry.getValue().getTag());
hostConfigMap.put(desiredConfigEntry.getKey(), hostConfig);
@@ -1344,7 +1357,7 @@ public class HostImpl implements Host {
if (configGroups != null && !configGroups.isEmpty()) {
for (ConfigGroup configGroup : configGroups.values()) {
for (Map.Entry<String, Config> configEntry : configGroup
- .getConfigurations().entrySet()) {
+ .getConfigurations().entrySet()) {
String configType = configEntry.getKey();
// HostConfig config holds configType -> versionTag, per config group
@@ -1364,7 +1377,7 @@ public class HostImpl implements Host {
}
Config config = configEntry.getValue();
hostConfig.getConfigGroupOverrides().put(configGroup.getId(),
- config.getTag());
+ config.getTag());
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/2bc94a04/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
index 86b119e..f79c823 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
@@ -31,7 +31,11 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Random;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
import javax.persistence.EntityManager;
@@ -224,6 +228,22 @@ public class ConfigHelperTest {
return configGroup.getId();
}
+ void applyConfig(Map<String, String> properties, String configType, String configTag) throws Exception {
+ ConfigurationRequest cr = new ConfigurationRequest();
+ cr.setClusterName(clusterName);
+ cr.setType(configType);
+ cr.setVersionTag(configTag);
+ cr.setProperties(properties);
+
+ final ClusterRequest clusterRequest =
+ new ClusterRequest(cluster.getClusterId(), clusterName,
+ cluster.getDesiredStackVersion().getStackVersion(), null);
+
+ clusterRequest.setDesiredConfig(Collections.singletonList(cr));
+ managementController.updateClusters(new HashSet<ClusterRequest>() {{
+ add(clusterRequest);
+ }}, null);
+ }
@Test
public void testEffectiveTagsForHost() throws Exception {
final Config config = new ConfigImpl("core-site");
@@ -745,6 +765,111 @@ public class ConfigHelperTest {
verify(sch);
}
+ @Test
+ public void testCalculateIsStaleConfigsParallel() throws Exception{
+ Map<String, HostConfig> schReturn = new HashMap<String, HostConfig>();
+ HostConfig hc = new HostConfig();
+ // Put a different version to check for change
+ hc.setDefaultVersionTag("version2");
+ schReturn.put("flume-conf", hc);
+
+ // set up mocks
+ final ServiceComponentHost sch = createNiceMock(ServiceComponentHost.class);
+ // set up expectations
+ expect(sch.getActualConfigs()).andReturn(schReturn).anyTimes();
+ expect(sch.getHostName()).andReturn("h1").anyTimes();
+ expect(sch.getClusterId()).andReturn(1l).anyTimes();
+ expect(sch.getServiceName()).andReturn("FLUME").anyTimes();
+ expect(sch.getServiceComponentName()).andReturn("FLUME_HANDLER").anyTimes();
+ replay(sch);
+ // Cluster level config changes
+
+ final Config config1 = cluster.getDesiredConfigByType("flume-conf");
+
+ applyConfig(new HashMap<String, String>(){{
+ put("property", "1");
+ }}, "flume-conf", "version2");
+
+ final Config config2 = cluster.getDesiredConfigByType("flume-conf");
+
+ applyConfig(new HashMap<String, String>(){{
+ put("property", "2");
+ }}, "flume-conf", "version3");
+
+ final Config config3 = cluster.getDesiredConfigByType("flume-conf");
+
+ cluster.addDesiredConfig("admin", new HashSet<Config>(){{add(config1);}});
+
+ final AtomicBoolean mustBeStale = new AtomicBoolean();
+ mustBeStale.set(false);
+
+ final AtomicBoolean failed = new AtomicBoolean();
+ failed.set(false);
+
+ final AtomicBoolean finished = new AtomicBoolean();
+ finished.set(false);
+ // avoid situations when not checked previous mustBeStale value yes and applied new config version
+ final Lock checkLock = new ReentrantLock();
+
+ // parallel thread that will compare actual stale config with expected accordingly to desired configs, checks if
+ // isStaleConfigs bypassing every cache and returns correct information.
+ Thread parallel = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ while(!finished.get()){
+ checkLock.lock();
+ try {
+ boolean isStale = configHelper.isStaleConfigs(sch);
+ if(mustBeStale.get() != isStale){
+ failed.set(true);
+ break;
+ }
+ } catch (AmbariException e) {
+ e.printStackTrace();
+ } finally {
+ checkLock.unlock();
+ }
+ }
+ }
+ });
+
+ parallel.start();
+
+ Random r = new Random();
+ for(int i=0; i< 1000; i++){
+ try {
+ checkLock.lock();
+ switch(r.nextInt(3)) {
+ case 0: {
+ cluster.addDesiredConfig("admin", new HashSet<Config>(){{add(config1);}});
+ mustBeStale.set(true);
+ checkLock.unlock();
+ break;
+ }
+ case 1: {
+ cluster.addDesiredConfig("admin", new HashSet<Config>(){{add(config2);}});
+ mustBeStale.set(false);
+ checkLock.unlock();
+ break;
+ }
+ case 2: {
+ cluster.addDesiredConfig("admin", new HashSet<Config>(){{add(config3);}});
+ mustBeStale.set(true);
+ checkLock.unlock();
+ break;
+ }
+ }
+ } catch (Exception e){
+ checkLock.unlock();
+ }
+ if(!parallel.isAlive()) {
+ break;
+ }
+ }
+ finished.set(true);
+ parallel.join();
+ Assert.assertFalse(failed.get());
+ }
}
public static class RunWithCustomModule {