[GitHub] [cloudstack] rhtyd commented on a change in pull request #3239: KVM: Fix agents dont reconnect post maintenance

[GitBox <gi...@apache.org>]

rhtyd commented on a change in pull request #3239: KVM: Fix agents dont reconnect post maintenance
URL: https://github.com/apache/cloudstack/pull/3239#discussion_r286913960
 
 

 ##########
 File path: server/src/com/cloud/resource/ResourceManagerImpl.java
 ##########
 @@ -2344,45 +2345,76 @@ private boolean doCancelMaintenance(final long hostId) {
             }
         }
 
+        handleAgentIfNotConnected(host, vms_migrating);
+
         try {
             resourceStateTransitTo(host, ResourceState.Event.AdminCancelMaintenance, _nodeId);
             _agentMgr.pullAgentOutMaintenance(hostId);
             retryHostMaintenance.remove(hostId);
+        } catch (final NoTransitionException e) {
+            s_logger.debug("Cannot transmit host " + host.getId() + "to Enabled state", e);
+            return false;
+        }
 
-            // for kvm, need to log into kvm host, restart cloudstack-agent
-            if ((host.getHypervisorType() == HypervisorType.KVM && !vms_migrating) || host.getHypervisorType() == HypervisorType.LXC) {
+        return true;
 
-                final boolean sshToAgent = Boolean.parseBoolean(_configDao.getValue(Config.KvmSshToAgentEnabled.key()));
-                if (!sshToAgent) {
-                    s_logger.info("Configuration tells us not to SSH into Agents. Please restart the Agent (" + hostId + ")  manually");
-                    return true;
-                }
+    }
 
-                _hostDao.loadDetails(host);
-                final String password = host.getDetail("password");
-                final String username = host.getDetail("username");
-                if (password == null || username == null) {
-                    s_logger.debug("Can't find password/username");
-                    return false;
-                }
-                final com.trilead.ssh2.Connection connection = SSHCmdHelper.acquireAuthorizedConnection(host.getPrivateIpAddress(), 22, username, password);
-                if (connection == null) {
-                    s_logger.debug("Failed to connect to host: " + host.getPrivateIpAddress());
-                    return false;
-                }
+    /**
+     * Handle agent (if available) if its not connected before cancelling maintenance.
+     * Agent must be connected before cancelling maintenance.
+     * If the host status is not Up:
+     * - If kvm.ssh.to.agent is true, then SSH into the host and restart the agent.
+     * - If kvm.shh.to.agent is false, then fail cancelling maintenance
+     */
+    protected void handleAgentIfNotConnected(HostVO host, boolean vmsMigrating) {
+        final boolean isAgentOnHost = host.getHypervisorType() == HypervisorType.KVM ||
+                host.getHypervisorType() == HypervisorType.LXC;
+        if (!isAgentOnHost || vmsMigrating || host.getStatus() == Status.Up) {
+            return;
+        }
+        final boolean sshToAgent = Boolean.parseBoolean(_configDao.getValue(KvmSshToAgentEnabled.key()));
+        if (sshToAgent) {
+            Pair<String, String> credentials = getHostCredentials(host);
+            connectAndRestartAgentOnHost(host, credentials.first(), credentials.second());
+        } else {
+            throw new CloudRuntimeException("SSH access is disabled, cannot cancel maintenance mode as " +
+                    "host agent is not connected");
+        }
+    }
 
-                try {
-                    SSHCmdHelper.SSHCmdResult result = SSHCmdHelper.sshExecuteCmdOneShot(connection, "service cloudstack-agent restart");
-                    s_logger.debug("cloudstack-agent restart result: " + result.toString());
-                } catch (final SshException e) {
-                    return false;
-                }
-            }
+    /**
+     * Get host credentials
+     * @throws CloudRuntimeException if username or password are not found
+     */
+    protected Pair<String, String> getHostCredentials(HostVO host) {
+        _hostDao.loadDetails(host);
+        final String password = host.getDetail("password");
+        final String username = host.getDetail("username");
+        if (password == null || username == null) {
+            throw new CloudRuntimeException("SSH to agent is enabled, but username/password credentials are not found");
+        }
+        return new Pair<>(username, password);
+    }
 
-            return true;
-        } catch (final NoTransitionException e) {
-            s_logger.debug("Cannot transmit host " + host.getId() + "to Enabled state", e);
-            return false;
+    /**
+     * True if agent is restarted via SSH. Assumes kvm.ssh.to.agent = true and host status is not Up
+     */
+    protected void connectAndRestartAgentOnHost(HostVO host, String username, String password) {
+        final com.trilead.ssh2.Connection connection = SSHCmdHelper.acquireAuthorizedConnection(
+                host.getPrivateIpAddress(), 22, username, password);
+        if (connection == null) {
+            throw new CloudRuntimeException("SSH to agent is enabled, but failed to connect to host: " + host.getPrivateIpAddress());
+        }
+        try {
+            SSHCmdHelper.SSHCmdResult result = SSHCmdHelper.sshExecuteCmdOneShot(
+                    connection, "service cloudstack-agent restart");
 
 Review comment:
   This may be changed to systemctl restart cloudstack-agent || service cloudstack-agent restart

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services