You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by vb...@apache.org on 2016/10/21 21:28:32 UTC

ambari git commit: AMBARI-18631. Incorporate database consistency check into main Ambari process.(vbrodetskyi)

Repository: ambari
Updated Branches:
  refs/heads/branch-2.5 49f58e840 -> 1b9073539


AMBARI-18631. Incorporate database consistency check into main Ambari process.(vbrodetskyi)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1b907353
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1b907353
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1b907353

Branch: refs/heads/branch-2.5
Commit: 1b907353991e8382ff47e30a5cb130412a6e3b08
Parents: 49f58e8
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Fri Oct 21 11:12:42 2016 +0300
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Fri Oct 21 11:12:42 2016 +0300

----------------------------------------------------------------------
 .../checks/DatabaseConsistencyCheckHelper.java  |  54 ++++----
 .../checks/DatabaseConsistencyChecker.java      |  16 +--
 .../ambari/server/controller/AmbariServer.java  |  51 ++++++--
 .../src/main/python/ambari_server_main.py       |  25 ++--
 .../DatabaseConsistencyCheckHelperTest.java     |   7 +-
 .../server/controller/AmbariServerTest.java     | 129 ++++++++++++++++++-
 6 files changed, 212 insertions(+), 70 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/1b907353/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
index 2d91eca..c60bf20 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
@@ -60,31 +60,23 @@ public class DatabaseConsistencyCheckHelper {
   private static DBAccessor dbAccessor;
 
 
-  private static boolean errorAvailable = false;
-  private static boolean warningAvailable = false;
+  private static boolean errorsFound = false;
+  private static boolean warningsFound = false;
 
-  public static boolean isErrorAvailable() {
-    return errorAvailable;
+  public static boolean ifErrorsFound() {
+    return errorsFound;
   }
 
-  public static void setErrorAvailable(boolean errorAvailable) {
-    errorAvailable = errorAvailable;
-  }
-
-  public static boolean isWarningAvailable() {
-    return warningAvailable;
-  }
-
-  public static void setWarningAvailable(boolean warningAvailable) {
-    warningAvailable = warningAvailable;
+  public static boolean ifWarningsFound() {
+    return warningsFound;
   }
 
   public static void resetErrorWarningFlags() {
-    errorAvailable = false;
-    warningAvailable = false;
+    errorsFound = false;
+    warningsFound = false;
   }
 
-  protected static void setInjector(Injector injector) {
+  public static void setInjector(Injector injector) {
     DatabaseConsistencyCheckHelper.injector = injector;
     // Clean up: new injector means static fields should be reinitalized, though in real life it only occurs during testing
     closeConnection();
@@ -111,6 +103,16 @@ public class DatabaseConsistencyCheckHelper {
     }
   }
 
+  public static void runAllDBChecks() {
+    LOG.info("******************************* Check database started *******************************");
+    checkForNotMappedConfigsToCluster();
+    checkForConfigsSelectedMoreThanOnce();
+    checkForHostsWithoutState();
+    checkHostComponentStatesCountEqualsHostComponentsDesiredStates();
+    checkServiceConfigs();
+    LOG.info("******************************* Check database completed *******************************");
+  }
+
   public static void checkDBVersionCompatible() throws AmbariException {
     LOG.info("Checking DB store version");
 
@@ -175,7 +177,7 @@ public class DatabaseConsistencyCheckHelper {
       }
       if (!nonSelectedConfigs.isEmpty()) {
         LOG.warn("You have config(s): {} that is(are) not mapped (in clusterconfigmapping table) to any cluster!", StringUtils.join(nonSelectedConfigs, ","));
-        warningAvailable = true;
+        warningsFound = true;
       }
     } catch (SQLException e) {
       LOG.error("Exception occurred during check for not mapped configs to cluster procedure: ", e);
@@ -233,7 +235,7 @@ public class DatabaseConsistencyCheckHelper {
         for (String clusterName : clusterConfigTypeMap.keySet()) {
           LOG.error("You have config(s), in cluster {}, that is(are) selected more than once in clusterconfigmapping table: {}",
                   clusterName ,StringUtils.join(clusterConfigTypeMap.get(clusterName), ","));
-          errorAvailable = true;
+          errorsFound = true;
         }
       }
 
@@ -288,7 +290,7 @@ public class DatabaseConsistencyCheckHelper {
 
         if (!hostsWithoutStatus.isEmpty()) {
           LOG.error("You have host(s) without state (in hoststate table): " + StringUtils.join(hostsWithoutStatus, ","));
-          errorAvailable = true;
+          errorsFound = true;
         }
       }
 
@@ -365,7 +367,7 @@ public class DatabaseConsistencyCheckHelper {
 
       if (hostComponentStateCount != hostComponentDesiredStateCount || hostComponentStateCount != mergedCount) {
         LOG.error("Your host component states (hostcomponentstate table) count not equals host component desired states (hostcomponentdesiredstate table) count!");
-        errorAvailable = true;
+        errorsFound = true;
       }
 
     } catch (SQLException e) {
@@ -455,7 +457,7 @@ public class DatabaseConsistencyCheckHelper {
 
         for (String clusterName : clusterServiceMap.keySet()) {
           LOG.warn("Service(s): {}, from cluster {} has no config(s) in serviceconfig table!", StringUtils.join(clusterServiceMap.get(clusterName), ","), clusterName);
-          warningAvailable = true;
+          warningsFound = true;
         }
 
       }
@@ -482,7 +484,7 @@ public class DatabaseConsistencyCheckHelper {
           Multimap<String, String> serviceVersion = clusterServiceVersionMap.get(clName);
           for (String servName : serviceVersion.keySet()) {
             LOG.error("In cluster {}, service config mapping is unavailable (in table serviceconfigmapping) for service {} with version(s) {}! ", clName, servName, StringUtils.join(serviceVersion.get(servName), ","));
-            errorAvailable = true;
+            errorsFound = true;
           }
         }
 
@@ -555,7 +557,7 @@ public class DatabaseConsistencyCheckHelper {
           } else {
             LOG.warn("Service {} is not available for stack {} in cluster {}",
                     serviceName, stackName + "-" + stackVersion, clusterName);
-            warningAvailable = true;
+            warningsFound = true;
           }
         }
 
@@ -573,7 +575,7 @@ public class DatabaseConsistencyCheckHelper {
                   if (!serviceConfigsFromStack.isEmpty()) {
                     LOG.error("Required config(s): {} is(are) not available for service {} with service config version {} in cluster {}",
                             StringUtils.join(serviceConfigsFromStack, ","), serviceName, Integer.toString(serviceVersion), clusterName);
-                    errorAvailable = true;
+                    errorsFound = true;
                   }
                 }
               }
@@ -610,7 +612,7 @@ public class DatabaseConsistencyCheckHelper {
         Multimap<String, String> serviceConfig = clusterServiceConfigType.get(clusterName);
         for (String serviceName : serviceConfig.keySet()) {
           LOG.error("You have non selected configs: {} for service {} from cluster {}!", StringUtils.join(serviceConfig.get(serviceName), ","), serviceName, clusterName);
-          errorAvailable = true;
+          errorsFound = true;
         }
       }
     } catch (SQLException e) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/1b907353/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyChecker.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyChecker.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyChecker.java
index 535d74f..f979b7e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyChecker.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyChecker.java
@@ -97,37 +97,27 @@ public class DatabaseConsistencyChecker {
   public static void main(String[] args) throws Exception {
     DatabaseConsistencyChecker databaseConsistencyChecker = null;
     try {
-      LOG.info("******************************* Check database started *******************************");
 
       Injector injector = Guice.createInjector(new CheckHelperControllerModule(), new CheckHelperAuditModule());
       databaseConsistencyChecker = injector.getInstance(DatabaseConsistencyChecker.class);
 
       databaseConsistencyChecker.startPersistenceService();
 
-      DatabaseConsistencyCheckHelper.checkForNotMappedConfigsToCluster();
-
-      DatabaseConsistencyCheckHelper.checkForConfigsSelectedMoreThanOnce();
-
-      DatabaseConsistencyCheckHelper.checkForHostsWithoutState();
-
-      DatabaseConsistencyCheckHelper.checkHostComponentStatesCountEqualsHostComponentsDesiredStates();
-
-      DatabaseConsistencyCheckHelper.checkServiceConfigs();
+      DatabaseConsistencyCheckHelper.runAllDBChecks();
 
       databaseConsistencyChecker.stopPersistenceService();
 
-      LOG.info("******************************* Check database completed *******************************");
     } catch (Throwable e) {
       if (e instanceof AmbariException) {
         LOG.error("Exception occurred during database check:", e);
         throw (AmbariException)e;
-      }else{
+      } else {
         LOG.error("Unexpected error, database check failed", e);
         throw new Exception("Unexpected error, database check failed", e);
       }
     } finally {
         DatabaseConsistencyCheckHelper.closeConnection();
-        if (DatabaseConsistencyCheckHelper.isErrorAvailable()) {
+        if (DatabaseConsistencyCheckHelper.ifErrorsFound()) {
           String ambariDBConsistencyCheckLog = "ambari-server-check-database.log";
           if (LOG instanceof Log4jLoggerAdapter) {
             org.apache.log4j.Logger dbConsistencyCheckHelperLogger = org.apache.log4j.Logger.getLogger(DatabaseConsistencyCheckHelper.class);

http://git-wip-us.apache.org/repos/asf/ambari/blob/1b907353/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
index a9c8fd0..c1ba7f5 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
@@ -19,6 +19,9 @@
 package org.apache.ambari.server.controller;
 
 
+import javax.crypto.BadPaddingException;
+import javax.servlet.DispatcherType;
+
 import java.io.File;
 import java.io.IOException;
 import java.net.Authenticator;
@@ -28,13 +31,8 @@ import java.net.URL;
 import java.util.EnumSet;
 import java.util.Enumeration;
 import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 import java.util.logging.LogManager;
 
-import javax.crypto.BadPaddingException;
-import javax.servlet.DispatcherType;
-
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.StateRecoveryManager;
 import org.apache.ambari.server.StaticallyInject;
@@ -147,7 +145,6 @@ import org.springframework.web.context.request.RequestContextListener;
 import org.springframework.web.context.support.GenericWebApplicationContext;
 import org.springframework.web.filter.DelegatingFilterProxy;
 
-import com.google.common.base.Joiner;
 import com.google.common.util.concurrent.ServiceManager;
 import com.google.gson.Gson;
 import com.google.inject.Guice;
@@ -159,6 +156,7 @@ import com.google.inject.name.Named;
 import com.google.inject.persist.Transactional;
 import com.sun.jersey.spi.container.servlet.ServletContainer;
 
+
 @Singleton
 public class AmbariServer {
   public static final String VIEWS_URL_PATTERN = "/api/v1/views/*";
@@ -303,6 +301,8 @@ public class AmbariServer {
 
     setSystemProperties(configs);
 
+    runDatabaseConsistencyCheck();
+
     try {
       ClassPathXmlApplicationContext parentSpringAppContext =
           new ClassPathXmlApplicationContext();
@@ -639,6 +639,39 @@ public class AmbariServer {
   }
 
   /**
+   * this method executes database consistency check if skip option was not added
+   */
+  protected void runDatabaseConsistencyCheck() throws Exception {
+    if (System.getProperty("skipDatabaseConsistencyCheck") == null) {
+      System.out.println("Database consistency check started");
+      Logger DB_CHECK_LOG = LoggerFactory.getLogger(DatabaseConsistencyCheckHelper.class);
+      try{
+        DatabaseConsistencyCheckHelper.runAllDBChecks();
+      } catch(Throwable e) {
+        System.out.println("Database consistency check: failed");
+        if (e instanceof AmbariException) {
+          DB_CHECK_LOG.error("Exception occurred during database check:", e);
+          System.out.println("Exception occurred during database check: " + e.getMessage());
+          e.printStackTrace();
+          throw (AmbariException)e;
+        } else {
+          DB_CHECK_LOG.error("Unexpected error, database check failed", e);
+          System.out.println("Unexpected error, database check failed: " + e.getMessage());
+          e.printStackTrace();
+          throw new Exception("Unexpected error, database check failed", e);
+        }
+      } finally {
+        if (DatabaseConsistencyCheckHelper.ifErrorsFound()) {
+          System.out.println("Database consistency check: failed");
+          System.exit(1);
+        } else {
+          System.out.println("Database consistency check: successful");
+        }
+      }
+    }
+  }
+
+  /**
    * installs bridge handler which redirects log entries from JUL to Slf4J
    */
   private void setupJulLogging() {
@@ -753,9 +786,9 @@ public class AmbariServer {
 
       gzipFilter.setInitParameter("methods", "GET,POST,PUT,DELETE");
       gzipFilter.setInitParameter("mimeTypes",
-          "text/html,text/plain,text/xml,text/css,application/x-javascript," +
-              "application/xml,application/x-www-form-urlencoded," +
-              "application/javascript,application/json");
+              "text/html,text/plain,text/xml,text/css,application/x-javascript," +
+                      "application/xml,application/x-www-form-urlencoded," +
+                      "application/javascript,application/json");
       gzipFilter.setInitParameter("minGzipSize", configs.getApiGzipMinSize());
     }
   }

http://git-wip-us.apache.org/repos/asf/ambari/blob/1b907353/ambari-server/src/main/python/ambari_server_main.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server_main.py b/ambari-server/src/main/python/ambari_server_main.py
index 57ec58d..313ff66 100644
--- a/ambari-server/src/main/python/ambari_server_main.py
+++ b/ambari-server/src/main/python/ambari_server_main.py
@@ -102,7 +102,7 @@ SERVER_START_CMD_DEBUG_WINDOWS = "{0} " \
     "org.apache.ambari.server.controller.AmbariServer"
 
 SERVER_INIT_TIMEOUT = 5
-SERVER_START_TIMEOUT = 10
+SERVER_START_TIMEOUT = 30
 
 SERVER_PING_TIMEOUT_WINDOWS = 5
 SERVER_PING_ATTEMPTS_WINDOWS = 4
@@ -207,6 +207,15 @@ def wait_for_server_start(pidFile, scmStatus):
   sys.stdout.write('\n')
   sys.stdout.flush()
 
+  if 'Database consistency check: failed' in open(configDefaults.SERVER_OUT_FILE).read():
+    print "DB configs consistency check failed. Run \"ambari-server start --skip-database-check\" to skip. " \
+    "If you use this \"--skip-database-check\" option, do not make any changes to your cluster topology " \
+    "or perform a cluster upgrade until you correct the database consistency issues. See " + \
+          configDefaults.DB_CHECK_LOG + "for more details on the consistency issues."
+  else:
+    print "DB consistency check: no errors were found."
+
+
   if found_pids <= 0:
     exitcode = check_exitcode(os.path.join(configDefaults.PID_DIR, EXITCODE_NAME))
     raise FatalException(-1, AMBARI_SERVER_DIE_MSG.format(exitcode, configDefaults.SERVER_OUT_FILE))
@@ -312,20 +321,6 @@ def server_process_main(options, scmStatus=None):
   else:
     print "Ambari database consistency check started..."
     properties.process_pair(CHECK_DATABASE_SKIPPED_PROPERTY, "false")
-    command = CHECK_DATABASE_HELPER_CMD.format(java_exe, class_path)
-
-    (retcode, stdout, stderr) = run_os_command(command, env=environ)
-
-    if retcode > 0:
-      print str(stdout)
-      raise FatalException(1, 'Database check failed to complete. Please check ' + configDefaults.SERVER_LOG_FILE +
-                            ' and ' + configDefaults.DB_CHECK_LOG + ' for more information.')
-    else:
-      print str(stdout)
-      print "Ambari database consistency check finished"
-
-      if not stdout.startswith("No errors"):
-        sys.exit(1)
 
   update_properties(properties)
   param_list = generate_child_process_param_list(ambari_user, java_exe, class_path, debug_start, suspend_mode)

http://git-wip-us.apache.org/repos/asf/ambari/blob/1b907353/ambari-server/src/test/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelperTest.java
index 4663310..f45d480 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelperTest.java
@@ -19,6 +19,7 @@ package org.apache.ambari.server.checks;
 
 
 import javax.persistence.EntityManager;
+import junit.framework.Assert;
 import static org.easymock.EasyMock.expect;
 
 import java.sql.Connection;
@@ -27,7 +28,6 @@ import java.sql.Statement;
 import java.util.HashMap;
 import java.util.Map;
 
-import junit.framework.Assert;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
 import org.apache.ambari.server.orm.DBAccessor;
 import org.apache.ambari.server.stack.StackManagerFactory;
@@ -35,7 +35,6 @@ import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.ServiceInfo;
 import org.apache.ambari.server.state.stack.OsFamily;
 import org.easymock.EasyMockSupport;
-import org.junit.Ignore;
 import org.junit.Test;
 
 import com.google.inject.AbstractModule;
@@ -380,8 +379,8 @@ public class DatabaseConsistencyCheckHelperTest {
     easyMockSupport.verifyAll();
 
     Assert.assertTrue("Missing service config for OPENSOFT R should have triggered a warning.",
-        DatabaseConsistencyCheckHelper.isWarningAvailable());
-    Assert.assertFalse("No errors should have been triggered.", DatabaseConsistencyCheckHelper.isErrorAvailable());
+        DatabaseConsistencyCheckHelper.ifWarningsFound());
+    Assert.assertFalse("No errors should have been triggered.", DatabaseConsistencyCheckHelper.ifErrorsFound());
   }
 
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/1b907353/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariServerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariServerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariServerTest.java
index 54f6147..45e319a 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariServerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariServerTest.java
@@ -18,7 +18,11 @@
 
 package org.apache.ambari.server.controller;
 
+import javax.persistence.EntityManager;
+import javax.servlet.DispatcherType;
+import javax.servlet.SessionCookieConfig;
 import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.anyString;
 import static org.easymock.EasyMock.createNiceMock;
 import static org.easymock.EasyMock.expect;
 import static org.easymock.EasyMock.replay;
@@ -27,17 +31,24 @@ import static org.easymock.EasyMock.verify;
 import java.net.Authenticator;
 import java.net.InetAddress;
 import java.net.PasswordAuthentication;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
 import java.util.EnumSet;
 
-import javax.servlet.DispatcherType;
-import javax.servlet.SessionCookieConfig;
-
 import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.checks.DatabaseConsistencyCheckHelper;
 import org.apache.ambari.server.configuration.Configuration;
+import org.apache.ambari.server.orm.DBAccessor;
 import org.apache.ambari.server.orm.GuiceJpaInitializer;
 import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
+import org.apache.ambari.server.stack.StackManagerFactory;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.stack.OsFamily;
 import org.apache.velocity.app.Velocity;
 import org.easymock.EasyMock;
+import org.easymock.EasyMockSupport;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.SessionManager;
 import org.eclipse.jetty.servlet.FilterHolder;
@@ -49,6 +60,7 @@ import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
+import com.google.inject.AbstractModule;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
 
@@ -191,4 +203,115 @@ public class AmbariServerTest {
     Assert.assertEquals(52, ((QueuedThreadPool) server.getThreadPool()).getMaxThreads());
 
   }
+
+  @Test
+  public void testRunDatabaseConsistencyCheck() throws Exception {
+    EasyMockSupport easyMockSupport = new EasyMockSupport();
+
+    final AmbariMetaInfo mockAmbariMetainfo = easyMockSupport.createNiceMock(AmbariMetaInfo.class);
+    final DBAccessor mockDBDbAccessor = easyMockSupport.createNiceMock(DBAccessor.class);
+    final Connection mockConnection = easyMockSupport.createNiceMock(Connection.class);
+    final Statement mockStatement = easyMockSupport.createNiceMock(Statement.class);
+    final OsFamily mockOSFamily = easyMockSupport.createNiceMock(OsFamily.class);
+    final StackManagerFactory mockStackManagerFactory = easyMockSupport.createNiceMock(StackManagerFactory.class);
+    final EntityManager mockEntityManager = easyMockSupport.createNiceMock(EntityManager.class);
+    final Clusters mockClusters = easyMockSupport.createNiceMock(Clusters.class);
+
+    AmbariServer ambariServer = new AmbariServer();
+
+
+    final Injector mockInjector = Guice.createInjector(new AbstractModule() {
+      @Override
+      protected void configure() {
+        bind(StackManagerFactory.class).toInstance(mockStackManagerFactory);
+        bind(AmbariMetaInfo.class).toInstance(mockAmbariMetainfo);
+        bind(DBAccessor.class).toInstance(mockDBDbAccessor);
+        bind(OsFamily.class).toInstance(mockOSFamily);
+        bind(EntityManager.class).toInstance(mockEntityManager);
+        bind(Clusters.class).toInstance(mockClusters);
+      }
+    });
+
+    expect(mockDBDbAccessor.getConnection()).andReturn(mockConnection).atLeastOnce();
+    expect(mockConnection.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE)).andReturn(mockStatement).atLeastOnce();
+    expect(mockStatement.executeQuery(anyString())).andReturn(null).atLeastOnce();
+
+    DatabaseConsistencyCheckHelper.setInjector(mockInjector);
+
+    easyMockSupport.replayAll();
+
+    mockAmbariMetainfo.init();
+
+    ambariServer.runDatabaseConsistencyCheck();
+
+    easyMockSupport.verifyAll();
+  }
+
+  @Test
+  public void testRunDatabaseConsistencyCheck_IgnoreDBCheck() throws Exception {
+    AmbariServer ambariServer = new AmbariServer();
+
+    System.setProperty("skipDatabaseConsistencyCheck", "");
+
+    final Injector mockInjector = Guice.createInjector(new AbstractModule() {
+      @Override
+      protected void configure() {
+
+      }
+    });
+
+    DatabaseConsistencyCheckHelper.setInjector(mockInjector);
+
+    ambariServer.runDatabaseConsistencyCheck();
+
+    System.clearProperty("skipDatabaseConsistencyCheck");
+  }
+
+  @Test
+  public void testRunDatabaseConsistencyCheck_ThrowException() throws Exception {
+    EasyMockSupport easyMockSupport = new EasyMockSupport();
+
+    final AmbariMetaInfo mockAmbariMetainfo = easyMockSupport.createNiceMock(AmbariMetaInfo.class);
+    final DBAccessor mockDBDbAccessor = easyMockSupport.createNiceMock(DBAccessor.class);
+    final OsFamily mockOSFamily = easyMockSupport.createNiceMock(OsFamily.class);
+    final StackManagerFactory mockStackManagerFactory = easyMockSupport.createNiceMock(StackManagerFactory.class);
+    final EntityManager mockEntityManager = easyMockSupport.createNiceMock(EntityManager.class);
+    final Clusters mockClusters = easyMockSupport.createNiceMock(Clusters.class);
+
+    AmbariServer ambariServer = new AmbariServer();
+
+
+    final Injector mockInjector = Guice.createInjector(new AbstractModule() {
+      @Override
+      protected void configure() {
+        bind(StackManagerFactory.class).toInstance(mockStackManagerFactory);
+        bind(AmbariMetaInfo.class).toInstance(mockAmbariMetainfo);
+        bind(DBAccessor.class).toInstance(mockDBDbAccessor);
+        bind(OsFamily.class).toInstance(mockOSFamily);
+        bind(EntityManager.class).toInstance(mockEntityManager);
+        bind(Clusters.class).toInstance(mockClusters);
+      }
+    });
+
+    expect(mockDBDbAccessor.getConnection()).andReturn(null);
+
+
+    DatabaseConsistencyCheckHelper.setInjector(mockInjector);
+
+    easyMockSupport.replayAll();
+
+    mockAmbariMetainfo.init();
+
+    boolean errorOccurred = false;
+    try {
+      ambariServer.runDatabaseConsistencyCheck();
+    } catch(Exception e) {
+      errorOccurred = true;
+    }
+
+    junit.framework.Assert.assertTrue(errorOccurred);
+
+    easyMockSupport.verifyAll();
+  }
+
 }