You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2015/12/29 12:33:42 UTC
ambari git commit: AMBARI-14385. Robust Handling of Ambari properites
and DB state. (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/trunk f697f4aff -> 65752c9bd
AMBARI-14385. Robust Handling of Ambari properites and DB state. (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/65752c9b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/65752c9b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/65752c9b
Branch: refs/heads/trunk
Commit: 65752c9bd95af7f64002e1b1da6b734b3bd99d8f
Parents: f697f4a
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Tue Dec 29 13:33:42 2015 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Tue Dec 29 13:33:42 2015 +0200
----------------------------------------------------------------------
.../ambari/server/controller/AmbariServer.java | 7 +-
.../controller/utilities/DatabaseChecker.java | 115 ++++++++++++++++++-
ambari-server/src/main/python/ambari-server.py | 4 +
.../python/ambari_server/serverConfiguration.py | 14 +++
.../src/main/python/ambari_server_main.py | 13 ++-
5 files changed, 148 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/65752c9b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
index c6f90c0..cc69134 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
@@ -260,8 +260,11 @@ public class AmbariServer {
server.setSessionIdManager(sessionIdManager);
Server serverForAgent = new Server();
- DatabaseChecker.checkDBVersion();
- DatabaseChecker.checkDBConsistency();
+ if (System.getProperty("skipDatabaseConsistencyValidation") == null) {
+ DatabaseChecker.checkDBVersion();
+ DatabaseChecker.checkDBConsistency();
+ DatabaseChecker.checkDBConfigsConsistency();
+ }
try {
ClassPathXmlApplicationContext parentSpringAppContext =
http://git-wip-us.apache.org/repos/asf/ambari/blob/65752c9b/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/DatabaseChecker.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/DatabaseChecker.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/DatabaseChecker.java
index c4a4e4c..4839d21 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/DatabaseChecker.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/utilities/DatabaseChecker.java
@@ -19,22 +19,31 @@
package org.apache.ambari.server.controller.utilities;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.orm.dao.ClusterDAO;
import org.apache.ambari.server.orm.dao.MetainfoDAO;
+import org.apache.ambari.server.orm.entities.ClusterConfigEntity;
+import org.apache.ambari.server.orm.entities.ClusterConfigMappingEntity;
import org.apache.ambari.server.orm.entities.ClusterEntity;
import org.apache.ambari.server.orm.entities.ClusterServiceEntity;
+import org.apache.ambari.server.orm.entities.ClusterStateEntity;
import org.apache.ambari.server.orm.entities.HostComponentDesiredStateEntity;
import org.apache.ambari.server.orm.entities.HostComponentStateEntity;
import org.apache.ambari.server.orm.entities.MetainfoEntity;
import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntity;
import org.apache.ambari.server.orm.entities.ServiceDesiredStateEntity;
+import org.apache.ambari.server.orm.entities.StackEntity;
import org.apache.ambari.server.state.ComponentInfo;
+import org.apache.ambari.server.state.ServiceInfo;
import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.state.StackInfo;
+import org.apache.ambari.server.state.State;
import org.apache.ambari.server.utils.VersionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -142,7 +151,108 @@ public class DatabaseChecker {
if (checkPassed) {
LOG.info("DB consistency check passed.");
} else {
- LOG.error("DB consistency check failed.");
+ String errorMessage = "DB consistency check failed. Run \"ambari-server start --skip-database-validation\" to skip validation.";
+ LOG.error(errorMessage);
+ throw new AmbariException(errorMessage);
+ }
+ }
+
+ private static boolean clusterConfigsContainTypeAndTag(Collection<ClusterConfigEntity> clusterConfigEntities,
+ String typeName, String tag) {
+ for (ClusterConfigEntity clusterConfigEntity : clusterConfigEntities) {
+ if (typeName.equals(clusterConfigEntity.getType()) && tag.equals(clusterConfigEntity.getTag())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Validates configuration consistency. Checks that every config type from stack is presented in
+ * ClusterConfigMapping. Checks that for each config type exactly one is selected. Checks that ClusterConfig
+ * contains type_names and tags from ClusterConfigMapping.
+ *
+ * @throws AmbariException if check failed
+ */
+ public static void checkDBConfigsConsistency() throws AmbariException {
+ LOG.info("Checking DB configs consistency");
+
+ boolean checkPassed = true;
+
+ if (ambariMetaInfo == null) {
+ ambariMetaInfo = injector.getInstance(AmbariMetaInfo.class);
+ }
+
+ ClusterDAO clusterDAO = injector.getInstance(ClusterDAO.class);
+ List<ClusterEntity> clusters = clusterDAO.findAll();
+ if (clusters != null) {
+ for (ClusterEntity clusterEntity : clusters) {
+ Collection<ClusterConfigMappingEntity> configMappingEntities = clusterEntity.getConfigMappingEntities();
+ Collection<ClusterConfigEntity> clusterConfigEntities = clusterEntity.getClusterConfigEntities();
+
+ if (configMappingEntities != null) {
+ Map<String, Integer> selectedCountForType = new HashMap<>();
+ for (ClusterConfigMappingEntity clusterConfigMappingEntity : configMappingEntities) {
+ String typeName = clusterConfigMappingEntity.getType();
+ if (clusterConfigMappingEntity.isSelected() > 0) {
+ int selectedCount = selectedCountForType.get(typeName) != null ? selectedCountForType.get(typeName) : 0;
+ selectedCountForType.put(typeName, selectedCount + 1);
+
+ // Check that ClusterConfig contains type_name and tag from ClusterConfigMapping
+ if (!clusterConfigsContainTypeAndTag(clusterConfigEntities, typeName, clusterConfigMappingEntity.getTag())) {
+ checkPassed = false;
+ LOG.error("ClusterConfig does not contain mapping for type_name=" + typeName + " tag="
+ + clusterConfigMappingEntity.getTag());
+ }
+ } else {
+ if (!selectedCountForType.containsKey(typeName)) {
+ selectedCountForType.put(typeName, 0);
+ }
+ }
+ }
+
+ // Check that every config type from stack is presented in ClusterConfigMapping
+ Collection<ClusterServiceEntity> clusterServiceEntities = clusterEntity.getClusterServiceEntities();
+ ClusterStateEntity clusterStateEntity = clusterEntity.getClusterStateEntity();
+ if (clusterStateEntity != null) {
+ StackEntity currentStack = clusterStateEntity.getCurrentStack();
+ StackInfo stack = ambariMetaInfo.getStack(currentStack.getStackName(), currentStack.getStackVersion());
+
+ for (ClusterServiceEntity clusterServiceEntity : clusterServiceEntities) {
+ if (!State.INIT.equals(clusterServiceEntity.getServiceDesiredStateEntity().getDesiredState())) {
+ String serviceName = clusterServiceEntity.getServiceName();
+ ServiceInfo serviceInfo = ambariMetaInfo.getService(stack.getName(), stack.getVersion(), serviceName);
+ for (String configTypeName : serviceInfo.getConfigTypeAttributes().keySet()) {
+ if (selectedCountForType.get(configTypeName) == null) {
+ checkPassed = false;
+ LOG.error("ClusterConfigMapping does not contain mapping for service=" + serviceName + " type_name="
+ + configTypeName);
+ } else {
+ // Check that for each config type exactly one is selected
+ if (selectedCountForType.get(configTypeName) == 0) {
+ checkPassed = false;
+ LOG.error("ClusterConfigMapping selected count is 0 for service=" + serviceName + " type_name="
+ + configTypeName);
+ } else if (selectedCountForType.get(configTypeName) > 1) {
+ checkPassed = false;
+ LOG.error("ClusterConfigMapping selected count is more than 1 for service=" + serviceName
+ + " type_name=" + configTypeName);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ if (checkPassed) {
+ LOG.info("DB configs consistency check passed.");
+ } else {
+ String errorMessage = "DB configs consistency check failed. Run \"ambari-server start --skip-database-validation\" to skip validation.";
+ LOG.error(errorMessage);
+ throw new AmbariException(errorMessage);
}
}
@@ -169,7 +279,8 @@ public class DatabaseChecker {
String error = "Current database store version is not compatible with " +
"current server version"
+ ", serverVersion=" + serverVersion
- + ", schemaVersion=" + schemaVersion;
+ + ", schemaVersion=" + schemaVersion
+ + " Run \"ambari-server start --skip-database-validation\" to skip validation";
LOG.warn(error);
throw new AmbariException(error);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/65752c9b/ambari-server/src/main/python/ambari-server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari-server.py b/ambari-server/src/main/python/ambari-server.py
index 41fdfb6..9f13ac5 100755
--- a/ambari-server/src/main/python/ambari-server.py
+++ b/ambari-server/src/main/python/ambari-server.py
@@ -316,6 +316,8 @@ def init_parser_options(parser):
help="Database user password")
parser.add_option('--jdbc-driver', default=None, dest="jdbc_driver",
help="Specifies the path to the JDBC driver JAR file")
+ parser.add_option('--skip-properties-validation', action="store_true", default=False, help="Skip properties file validation", dest="skip_properties_validation")
+ parser.add_option('--skip-database-validation', action="store_true", default=False, help="Skip database consistency validation", dest="skip_database_validation")
# -b and -i the remaining available short options
# -h reserved for help
@@ -380,6 +382,8 @@ def init_parser_options(parser):
dest="jdbc_db")
parser.add_option('--cluster-name', default=None, help="Cluster name", dest="cluster_name")
parser.add_option('--version-display-name', default=None, help="Display name of desired repo version", dest="desired_repo_version")
+ parser.add_option('--skip-properties-validation', action="store_true", default=False, help="Skip properties file validation", dest="skip_properties_validation")
+ parser.add_option('--skip-database-validation', action="store_true", default=False, help="Skip database consistency validation", dest="skip_database_validation")
parser.add_option('--force-version', action="store_true", default=False, help="Force version to current", dest="force_repo_version")
parser.add_option('--version', dest="stack_versions", default=None, action="append", type="string",
help="Specify stack version that needs to be enabled. All other stacks versions will be disabled")
http://git-wip-us.apache.org/repos/asf/ambari/blob/65752c9b/ambari-server/src/main/python/ambari_server/serverConfiguration.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/serverConfiguration.py b/ambari-server/src/main/python/ambari_server/serverConfiguration.py
index 223c608..aeb4d5b 100644
--- a/ambari-server/src/main/python/ambari_server/serverConfiguration.py
+++ b/ambari-server/src/main/python/ambari_server/serverConfiguration.py
@@ -177,6 +177,11 @@ SETUP_OR_UPGRADE_MSG = "- If this is a new setup, then run the \"ambari-server s
DEFAULT_DB_NAME = "ambari"
+COMMON_SERVICES_PATH_PROPERTY = 'common.services.path'
+WEBAPP_DIR_PROPERTY = 'webapp.dir'
+REQUIRED_PROPERTIES = [OS_FAMILY_PROPERTY, OS_TYPE_PROPERTY, COMMON_SERVICES_PATH_PROPERTY, SERVER_VERSION_FILE_PATH, \
+ WEBAPP_DIR_PROPERTY, STACK_LOCATION_KEY]
+
class ServerDatabaseType(object):
internal = 0
remote = 1
@@ -1303,3 +1308,12 @@ def get_stack_location(properties):
if stack_location is None:
stack_location = configDefaults.STACK_LOCATION_DEFAULT
return stack_location
+
+def get_missing_properties(properties):
+ missing_propertiers = []
+ for property in REQUIRED_PROPERTIES:
+ value = properties[property]
+ if not value:
+ missing_propertiers.append(property)
+
+ return missing_propertiers
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/65752c9b/ambari-server/src/main/python/ambari_server_main.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server_main.py b/ambari-server/src/main/python/ambari_server_main.py
index cf8e8c0..5c98bf4 100644
--- a/ambari-server/src/main/python/ambari_server_main.py
+++ b/ambari-server/src/main/python/ambari_server_main.py
@@ -32,7 +32,7 @@ from ambari_server.serverConfiguration import configDefaults, find_jdk, get_amba
get_conf_dir, get_is_persisted, get_is_secure, get_java_exe_path, get_original_master_key, read_ambari_user, \
get_is_active_instance, \
PID_NAME, SECURITY_KEY_ENV_VAR_NAME, SECURITY_MASTER_KEY_LOCATION, \
- SETUP_OR_UPGRADE_MSG, check_database_name_property, parse_properties_file
+ SETUP_OR_UPGRADE_MSG, check_database_name_property, parse_properties_file, get_missing_properties
from ambari_server.serverUtils import refresh_stack_hash
from ambari_server.setupHttps import get_fqdn
from ambari_server.setupSecurity import generate_env, \
@@ -237,6 +237,13 @@ def server_process_main(options, scmStatus=None):
properties = get_ambari_properties()
+ if not options.skip_properties_validation:
+ missing_properties = get_missing_properties(properties)
+ if missing_properties:
+ err = "Required properties are not found: " + str(missing_properties) + ". To skip properties validation " \
+ "use \"--skip-properties-validation\""
+ raise FatalException(1, err)
+
# Preparations
if is_root():
print configDefaults.MESSAGE_SERVER_RUNNING_AS_ROOT
@@ -267,6 +274,10 @@ def server_process_main(options, scmStatus=None):
suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
suspend_mode = 'y' if suspend_start else 'n'
+ if options.skip_database_validation:
+ global jvm_args
+ jvm_args += " -DskipDatabaseConsistencyValidation"
+
param_list = generate_child_process_param_list(ambari_user, java_exe,
serverClassPath.get_full_ambari_classpath_escaped_for_shell(), debug_start,
suspend_mode)