You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by vb...@apache.org on 2017/02/03 14:05:09 UTC
ambari git commit: AMBARI-19856. Perf: start/stop all actions works
much slower after few days of testing.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk f2561c48f -> 132e26652
AMBARI-19856. Perf: start/stop all actions works much slower after few days of testing.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/132e2665
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/132e2665
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/132e2665
Branch: refs/heads/trunk
Commit: 132e266521642d90908d3dd69446a701ca94c3db
Parents: f2561c4
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Fri Feb 3 16:04:21 2017 +0200
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Fri Feb 3 16:04:21 2017 +0200
----------------------------------------------------------------------
.../AmbariManagementControllerImpl.java | 22 +++++++------
.../ambari/server/state/ConfigHelper.java | 33 ++++++++++++++------
.../server/state/ServiceComponentHost.java | 5 +++
.../svccomphost/ServiceComponentHostImpl.java | 21 ++++++++++---
contrib/utils/perf/deploy-gce-perf-cluster.py | 11 +++++--
5 files changed, 67 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/132e2665/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index ce9d7a9..9314c28 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -2194,7 +2194,8 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
boolean skipFailure,
ClusterVersionEntity effectiveClusterVersion,
boolean isUpgradeSuspended,
- DatabaseType databaseType
+ DatabaseType databaseType,
+ Map<String, DesiredConfig> clusterDesiredConfigs
)
throws AmbariException {
@@ -2398,19 +2399,17 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
String packageList = gson.toJson(packages);
hostParams.put(PACKAGE_LIST, packageList);
- Map<String, DesiredConfig> desiredConfigs = cluster.getDesiredConfigs();
-
Set<PropertyInfo> stackProperties = ambariMetaInfo.getStackProperties(stackInfo.getName(), stackInfo.getVersion());
- Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(PropertyType.USER, cluster, desiredConfigs, servicesMap, stackProperties);
+ Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(PropertyType.USER, cluster, clusterDesiredConfigs, servicesMap, stackProperties);
String userList = gson.toJson(userSet);
hostParams.put(USER_LIST, userList);
- Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(PropertyType.GROUP, cluster, desiredConfigs, servicesMap, stackProperties);
+ Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(PropertyType.GROUP, cluster, clusterDesiredConfigs, servicesMap, stackProperties);
String groupList = gson.toJson(groupSet);
hostParams.put(GROUP_LIST, groupList);
- Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs, servicesMap, stackProperties);
+ Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(PropertyType.NOT_MANAGED_HDFS_PATH, cluster, clusterDesiredConfigs, servicesMap, stackProperties);
String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet);
hostParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList);
@@ -2900,9 +2899,11 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
configurationAttributes =
new TreeMap<>();
Host host = clusters.getHost(scHost.getHostName());
+ Map<String, DesiredConfig> clusterDesiredConfigs = cluster.getDesiredConfigs();
Map<String, Map<String, String>> configTags =
- findConfigurationTagsWithOverrides(cluster, host.getHostName());
+ configHelper.getEffectiveDesiredTags(cluster, host.getHostName(), clusterDesiredConfigs);
+
// Skip INSTALL task in case SysPrepped hosts and in case of server components. In case of server component
// START task should run configuration script.
@@ -2911,7 +2912,8 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
scHost.setState(State.INSTALLED);
} else {
createHostAction(cluster, stage, scHost, configurations, configurationAttributes, configTags,
- roleCommand, requestParameters, event, skipFailure, effectiveClusterVersion, isUpgradeSuspended, databaseType);
+ roleCommand, requestParameters, event, skipFailure, effectiveClusterVersion, isUpgradeSuspended,
+ databaseType, clusterDesiredConfigs);
}
}
@@ -3050,8 +3052,10 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
ClusterVersionEntity effectiveClusterVersion = cluster.getEffectiveClusterVersion();
boolean isUpgradeSuspended = cluster.isUpgradeSuspended();
DatabaseType databaseType = configs.getDatabaseType();
+ Map<String, DesiredConfig> clusterDesiredConfigs = cluster.getDesiredConfigs();
createHostAction(cluster, stage, scHost, configurations, configurationAttributes, configTags,
- roleCommand, null, null, false, effectiveClusterVersion, isUpgradeSuspended, databaseType);
+ roleCommand, null, null, false, effectiveClusterVersion, isUpgradeSuspended, databaseType,
+ clusterDesiredConfigs);
ExecutionCommand ec = stage.getExecutionCommands().get(scHost.getHostName()).get(0).getExecutionCommand();
// createHostAction does not take a hostLevelParams but creates one
http://git-wip-us.apache.org/repos/asf/ambari/blob/132e2665/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
index 82b41b5..0e5b434 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
@@ -37,6 +37,7 @@ import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.orm.dao.ClusterDAO;
import org.apache.ambari.server.orm.entities.ClusterConfigEntity;
+import org.apache.ambari.server.orm.entities.HostComponentDesiredStateEntity;
import org.apache.ambari.server.state.PropertyInfo.PropertyType;
import org.apache.ambari.server.state.configgroup.ConfigGroup;
import org.apache.ambari.server.utils.SecretReference;
@@ -435,10 +436,17 @@ public class ConfigHelper {
*/
public boolean isStaleConfigs(ServiceComponentHost sch, Map<String, DesiredConfig> requestDesiredConfigs)
throws AmbariException {
- boolean stale = calculateIsStaleConfigs(sch, requestDesiredConfigs);
+ HostComponentDesiredStateEntity hostComponentDesiredStateEntity = sch.getDesiredStateEntity();
+ return isStaleConfigs(sch, requestDesiredConfigs, hostComponentDesiredStateEntity);
+ }
+
+ public boolean isStaleConfigs(ServiceComponentHost sch, Map<String, DesiredConfig> requestDesiredConfigs,
+ HostComponentDesiredStateEntity hostComponentDesiredStateEntity)
+ throws AmbariException {
+ boolean stale = calculateIsStaleConfigs(sch, requestDesiredConfigs, hostComponentDesiredStateEntity);
if (LOG.isDebugEnabled()) {
LOG.debug("Cache configuration staleness for host {} and component {} as {}",
- sch.getHostName(), sch.getServiceComponentName(), stale);
+ sch.getHostName(), sch.getServiceComponentName(), stale);
}
return stale;
}
@@ -1010,7 +1018,14 @@ public class ConfigHelper {
private boolean calculateIsStaleConfigs(ServiceComponentHost sch,
Map<String, DesiredConfig> desiredConfigs) throws AmbariException {
- if (sch.isRestartRequired()) {
+ HostComponentDesiredStateEntity hostComponentDesiredStateEntity = sch.getDesiredStateEntity();
+ return calculateIsStaleConfigs(sch, desiredConfigs, hostComponentDesiredStateEntity);
+ }
+
+ private boolean calculateIsStaleConfigs(ServiceComponentHost sch, Map<String, DesiredConfig> desiredConfigs,
+ HostComponentDesiredStateEntity hostComponentDesiredStateEntity) throws AmbariException {
+
+ if (sch.isRestartRequired(hostComponentDesiredStateEntity)) {
return true;
}
@@ -1022,16 +1037,16 @@ public class ConfigHelper {
Cluster cluster = clusters.getClusterById(sch.getClusterId());
Map<String, Map<String, String>> desired = getEffectiveDesiredTags(cluster, sch.getHostName(),
- desiredConfigs);
+ desiredConfigs);
Boolean stale = null;
int staleHash = 0;
if (STALE_CONFIGS_CACHE_ENABLED){
staleHash = Objects.hashCode(actual.hashCode(),
- desired.hashCode(),
- sch.getHostName(),
- sch.getServiceComponentName(),
- sch.getServiceName());
+ desired.hashCode(),
+ sch.getHostName(),
+ sch.getServiceComponentName(),
+ sch.getServiceName());
stale = staleConfigsCache.getIfPresent(staleHash);
if(stale != null) {
return stale;
@@ -1043,7 +1058,7 @@ public class ConfigHelper {
StackId stackId = cluster.getDesiredStackVersion();
ServiceInfo serviceInfo = ambariMetaInfo.getService(stackId.getStackName(),
- stackId.getStackVersion(), sch.getServiceName());
+ stackId.getStackVersion(), sch.getServiceName());
ComponentInfo componentInfo = serviceInfo.getComponentByName(sch.getServiceComponentName());
// Configs are considered stale when:
http://git-wip-us.apache.org/repos/asf/ambari/blob/132e2665/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentHost.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentHost.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentHost.java
index fd92bed..104e456 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentHost.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentHost.java
@@ -23,6 +23,7 @@ import java.util.Map;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.controller.ServiceComponentHostResponse;
+import org.apache.ambari.server.orm.entities.HostComponentDesiredStateEntity;
import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.state.fsm.InvalidStateTransitionException;
@@ -234,6 +235,8 @@ public interface ServiceComponentHost {
*/
boolean isRestartRequired();
+ boolean isRestartRequired(HostComponentDesiredStateEntity hostComponentDesiredStateEntity);
+
/**
* @param restartRequired the restartRequired flag
*/
@@ -246,4 +249,6 @@ public interface ServiceComponentHost {
*/
RepositoryVersionEntity recalculateHostVersionState() throws AmbariException;
+ HostComponentDesiredStateEntity getDesiredStateEntity();
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/132e2665/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
index 95de4e8..6f01048 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
@@ -1214,10 +1214,14 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
@Override
public HostComponentAdminState getComponentAdminState() {
HostComponentDesiredStateEntity desiredStateEntity = getDesiredStateEntity();
+ return getComponentAdminStateFromDesiredStateEntity(desiredStateEntity);
+ }
+
+ private HostComponentAdminState getComponentAdminStateFromDesiredStateEntity(HostComponentDesiredStateEntity desiredStateEntity) {
if (desiredStateEntity != null) {
HostComponentAdminState adminState = desiredStateEntity.getAdminState();
if (adminState == null && !serviceComponent.isClientComponent()
- && !serviceComponent.isMasterComponent()) {
+ && !serviceComponent.isMasterComponent()) {
adminState = HostComponentAdminState.INSERVICE;
}
return adminState;
@@ -1245,6 +1249,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
@Override
public ServiceComponentHostResponse convertToResponse(Map<String, DesiredConfig> desiredConfigs) {
HostComponentStateEntity hostComponentStateEntity = getStateEntity();
+ HostEntity hostEntity = hostComponentStateEntity.getHostEntity();
if (null == hostComponentStateEntity) {
LOG.warn(
"Could not convert ServiceComponentHostResponse to a response. It's possible that Host {} was deleted.",
@@ -1259,12 +1264,12 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
String serviceName = serviceComponent.getServiceName();
String serviceComponentName = serviceComponent.getName();
String hostName = getHostName();
- String publicHostName = getPublicHostName();
+ String publicHostName = hostEntity.getPublicHostName();
String state = getState().toString();
String stackId = stackVersion.getStackId();
String desiredState = (hostComponentDesiredStateEntity == null) ? null : hostComponentDesiredStateEntity.getDesiredState().toString();
String desiredStackId = getDesiredStackVersionFromHostComponentDesiredStateEntity(hostComponentDesiredStateEntity).getStackId();
- HostComponentAdminState componentAdminState = getComponentAdminState();
+ HostComponentAdminState componentAdminState = getComponentAdminStateFromDesiredStateEntity(hostComponentDesiredStateEntity);
UpgradeState upgradeState = hostComponentStateEntity.getUpgradeState();
String displayName = null;
@@ -1284,7 +1289,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
r.setUpgradeState(upgradeState);
try {
- r.setStaleConfig(helper.isStaleConfigs(this, desiredConfigs));
+ r.setStaleConfig(helper.isStaleConfigs(this, desiredConfigs, hostComponentDesiredStateEntity));
} catch (Exception e) {
LOG.error("Could not determine stale config", e);
}
@@ -1503,6 +1508,11 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
}
@Override
+ public boolean isRestartRequired(HostComponentDesiredStateEntity hostComponentDesiredStateEntity) {
+ return hostComponentDesiredStateEntity.isRestartRequired();
+ }
+
+ @Override
public void setRestartRequired(boolean restartRequired) {
LOG.debug("Set RestartRequired on serviceName = {} componentName = {} hostName = {} to {}",
getServiceName(), getServiceComponentName(), getHostName(), restartRequired);
@@ -1588,7 +1598,8 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
*
* @return
*/
- private HostComponentDesiredStateEntity getDesiredStateEntity() {
+ @Override
+ public HostComponentDesiredStateEntity getDesiredStateEntity() {
return hostComponentDesiredStateDAO.findById(desiredStateEntityId);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/132e2665/contrib/utils/perf/deploy-gce-perf-cluster.py
----------------------------------------------------------------------
diff --git a/contrib/utils/perf/deploy-gce-perf-cluster.py b/contrib/utils/perf/deploy-gce-perf-cluster.py
index 7e84c40..6364122 100644
--- a/contrib/utils/perf/deploy-gce-perf-cluster.py
+++ b/contrib/utils/perf/deploy-gce-perf-cluster.py
@@ -339,8 +339,10 @@ def create_server_script(server_host_name):
"\n" + \
"\n" + \
"yum install mysql-connector-java* -y\n" + \
- "yum install mysql-server -y\n" + \
- "sed -i -e 's/mysqld]/mysqld]\\nmax_allowed_packet=16M/1' /etc/my.cnf\n" + \
+ "cd /etc/yum.repos.d/; wget http://repo.mysql.com/mysql-community-release-el6-5.noarch.rpm; rpm -ivh mysql-community-release-el6-5.noarch.rpm;" + \
+ "yum clean all; yum install mysql-server -y\n" + \
+ "sed -i -e 's/mysqld]/mysqld]\\nmax_allowed_packet=1024M\\njoin_buffer_size=512M\\nsort_buffer_size=128M\\nread_rnd_buffer_size=128M\\ninnodb_buffer_pool_size=16G" \
+ "\\ninnodb_file_io_threads=16\\ninnodb_thread_concurrency=32\\nkey_buffer_size=16G\\nquery_cache_limit=16M\\nquery_cache_size=512M\\nthread_cache_size=128\\ninnodb_log_buffer_size=512M/1' /etc/my.cnf\n" + \
"service mysqld start\n" + \
"mysql -uroot -e \"CREATE DATABASE ambari;\"\n" + \
"mysql -uroot -e \"SOURCE /var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql;\" ambari\n" + \
@@ -359,6 +361,8 @@ def create_server_script(server_host_name):
"sed -i -e 's/server.persistence.type=local/server.persistence.type=remote/g' /etc/ambari-server/conf/ambari.properties\n" + \
"sed -i -e 's/local.database.user=postgres//g' /etc/ambari-server/conf/ambari.properties\n" + \
"sed -i -e 's/server.jdbc.postgres.schema=ambari//g' /etc/ambari-server/conf/ambari.properties\n" + \
+ "sed -i -e 's/agent.threadpool.size.max=25/agent.threadpool.size.max=100/g' /etc/ambari-server/conf/ambari.properties\n" + \
+ "sed -i -e 's/client.threadpool.size.max=25/client.threadpool.size.max=65/g' /etc/ambari-server/conf/ambari.properties\n" + \
"sed -i -e 's/false/true/g' /var/lib/ambari-server/resources/stacks/PERF/1.0/metainfo.xml\n" + \
"sed -i -e 's/false/true/g' /var/lib/ambari-server/resources/stacks/PERF/2.0/metainfo.xml\n" + \
"sed -i -e 's/-Xmx2048m/-Xmx16384m/g' /var/lib/ambari-server/ambari-env.sh\n" + \
@@ -370,6 +374,9 @@ def create_server_script(server_host_name):
"echo 'server.jdbc.port=3306' >> /etc/ambari-server/conf/ambari.properties\n" + \
"echo 'server.jdbc.hostname=localhost' >> /etc/ambari-server/conf/ambari.properties\n" + \
"echo 'server.jdbc.driver.path=/usr/share/java/mysql-connector-java.jar' >> /etc/ambari-server/conf/ambari.properties\n" + \
+ "echo 'alerts.cache.enabled=true' >> /etc/ambari-server/conf/ambari.properties\n" + \
+ "echo 'alerts.cache.size=100000' >> /etc/ambari-server/conf/ambari.properties\n" + \
+ "echo 'alerts.execution.scheduler.maxThreads=4' >> /etc/ambari-server/conf/ambari.properties\n" + \
"echo 'security.temporary.keystore.retention.minutes=180' >> /etc/ambari-server/conf/ambari.properties\n" + \
"\n" + \
"ambari-server start --skip-database-check\n" + \