You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2017/11/13 16:57:39 UTC
[22/51] [abbrv] ambari git commit: AMBARI-22337 each service should
be able to implement server actions,
package them add a jar to be loaded during EU (dili)
AMBARI-22337 each service should be able to implement server actions, package them add a jar to be loaded during EU (dili)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/063ba36e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/063ba36e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/063ba36e
Branch: refs/heads/branch-feature-AMBARI-21674
Commit: 063ba36ef5d2ae2f407b46aae3f993ab81eae310
Parents: 38f6740
Author: Di Li <di...@apache.org>
Authored: Tue Nov 7 10:51:48 2017 -0500
Committer: Di Li <di...@apache.org>
Committed: Tue Nov 7 10:51:48 2017 -0500
----------------------------------------------------------------------
ambari-server/pom.xml | 12 ++
.../serveraction/ServerActionExecutor.java | 147 +++++++++++++++++--
.../ambari/server/stack/ServiceDirectory.java | 29 ++++
.../ambari/server/stack/ServiceModule.java | 8 +
.../apache/ambari/server/state/ServiceInfo.java | 14 ++
.../ambari/server/stack/ServiceModuleTest.java | 30 ++++
.../server/stack/StackManagerExtensionTest.java | 6 +
7 files changed, 232 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/063ba36e/ambari-server/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml
index a86acf5..2ecf435 100644
--- a/ambari-server/pom.xml
+++ b/ambari-server/pom.xml
@@ -198,6 +198,18 @@
</target>
</configuration>
</execution>
+ <execution>
+ <id>generate-test-oozie2-server-actions-dir</id>
+ <phase>process-test-classes</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <mkdir dir="target/test-classes/extensions/EXT/0.1/services/OOZIE2/server_actions/tmp"/>
+ </target>
+ </configuration>
+ </execution>
</executions>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/ambari/blob/063ba36e/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java
index 50e3cfe..e219dc3 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java
@@ -18,10 +18,17 @@
package org.apache.ambari.server.serveraction;
+import java.io.File;
+import java.io.FilenameFilter;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.StringTokenizer;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.ConcurrentHashMap;
@@ -38,11 +45,17 @@ import org.apache.ambari.server.actionmanager.HostRoleStatus;
import org.apache.ambari.server.actionmanager.Stage;
import org.apache.ambari.server.agent.CommandReport;
import org.apache.ambari.server.agent.ExecutionCommand;
+import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.configuration.Configuration;
+import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.security.authorization.internal.InternalAuthenticationToken;
+import org.apache.ambari.server.state.ServiceInfo;
+import org.apache.ambari.server.state.UpgradeContext.UpgradeServiceSummary;
+import org.apache.ambari.server.state.UpgradeContext.UpgradeSummary;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.util.ClassUtils;
import com.google.inject.Inject;
import com.google.inject.Injector;
@@ -495,7 +508,6 @@ public class ServerActionExecutor {
throw new AmbariException("Missing ExecutionCommand data");
} else {
Map<String, String> roleParams = executionCommand.getRoleParams();
-
if (roleParams == null) {
throw new AmbariException("Missing RoleParams data");
} else {
@@ -504,8 +516,30 @@ public class ServerActionExecutor {
if (actionClassname == null) {
throw new AmbariException("Missing action classname for server action");
} else {
- ServerAction action = createServerAction(actionClassname);
-
+ Map<String, ServiceInfo> services = new HashMap<String, ServiceInfo>();
+ UpgradeSummary upgradeSummary = executionCommand.getUpgradeSummary();
+ if (upgradeSummary != null) {
+ Map<String, UpgradeServiceSummary> upgradeServiceSummaries = upgradeSummary.services;
+ LOG.debug("UpgradeServiceSummary: " + upgradeServiceSummaries);
+ AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+ AmbariMetaInfo ambariMetaInfo = ambariManagementController.getAmbariMetaInfo();
+ String serviceName = executionCommand.getServiceName();
+ if (serviceName != null && !serviceName.isEmpty()){
+ LOG.info(String.format("Server action %s is associated with service %s", actionClassname, serviceName));
+ //Execution stage of a given service, only need to examine stack information for this one service
+ UpgradeServiceSummary serviceSummary = upgradeServiceSummaries.get(serviceName);
+ addServiceInfo(services, ambariMetaInfo, serviceSummary.sourceStackId, serviceName);
+ } else {
+ LOG.info(String.format("Server action %s is not associated with a service", actionClassname));
+ //Load all Jars
+ for(String key: upgradeServiceSummaries.keySet()){
+ UpgradeServiceSummary serviceSummary = upgradeServiceSummaries.get(key);
+ addServiceInfo(services, ambariMetaInfo, serviceSummary.sourceStackId, key);
+ }
+ }
+ LOG.info(String.format("Attempt to load server action classes from %s", services.keySet().toString()));
+ }
+ ServerAction action = createServerAction(actionClassname, services);
if (action == null) {
throw new AmbariException("Failed to create server action: " + actionClassname);
} else {
@@ -520,6 +554,30 @@ public class ServerActionExecutor {
}
}
+ private void addServiceInfo(Map<String, ServiceInfo> services, AmbariMetaInfo ambariMetaInfo, String stackId, String serviceName) {
+ List<String> stackInfo = getStackInfo(stackId);
+ LOG.debug(String.format("Stack info list: %s", stackInfo));
+ if (stackInfo.size() > 1) {
+ try {
+ ServiceInfo service = ambariMetaInfo.getService(stackInfo.get(0), stackInfo.get(1), serviceName);
+ LOG.debug(String.format("Adding %s to the list of services for loading external Jars...", service.getName()));
+ services.put(serviceName, service);
+ } catch (AmbariException e) {
+ LOG.error(String.format("Failed to obtain service info for stack %s, service name %s", stackId, serviceName), e);
+ }
+ }
+ }
+
+ private List<String> getStackInfo(String stackId) {
+ LOG.debug(String.format("Stack id: %s", stackId));
+ StringTokenizer tokens = new StringTokenizer(stackId, "-");
+ List<String> info = new ArrayList<String>();
+ while (tokens.hasMoreElements()) {
+ info.add((String)tokens.nextElement());
+ }
+ return info;
+ }
+
/**
* Attempts to create an instance of the ServerAction class implementation specified in
* classname.
@@ -528,24 +586,85 @@ public class ServerActionExecutor {
* @return the instantiated ServerAction implementation
* @throws AmbariException
*/
- private ServerAction createServerAction(String classname) throws AmbariException {
- try {
- Class<?> actionClass = Class.forName(classname);
+ private ServerAction createServerAction(String classname, Map<String, ServiceInfo> services) throws AmbariException {
+ Class<?> actionClass = null;
+ actionClass = getServerActionClass(classname);
+ if (actionClass == null) {
+ LOG.debug(String.format("Did not find %s in Ambari, try to load it from external directories", classname));
+ actionClass = getServiceLevelServerActionClass(classname, services);
+ }
- if (actionClass == null) {
- throw new AmbariException("Unable to load server action class: " + classname);
+ if (actionClass == null) {
+ throw new AmbariException("Unable to load server action class: " + classname);
+ } else {
+ LOG.debug(String.format("Ready to init server action %s", classname));
+ Class<? extends ServerAction> serverActionClass = actionClass.asSubclass(ServerAction.class);
+ if (serverActionClass == null) {
+ throw new AmbariException("Unable to execute server action class, invalid type: " + classname);
} else {
- Class<? extends ServerAction> serverActionClass = actionClass.asSubclass(ServerAction.class);
+ return injector.getInstance(serverActionClass);
+ }
+ }
+ }
- if (serverActionClass == null) {
- throw new AmbariException("Unable to execute server action class, invalid type: " + classname);
- } else {
- return injector.getInstance(serverActionClass);
+ /**
+ * Load server action classes defined in the service level Jar files
+ * */
+ private Class<?> getServiceLevelServerActionClass(String classname, Map<String, ServiceInfo> services) {
+ List<URL> urls = new ArrayList<>();
+ for (ServiceInfo service : services.values()) {
+ LOG.debug(String.format("Checking service %s", service));
+ File dir = service.getServerActionsFolder();
+ if ( dir != null) {
+ LOG.debug(String.format("Service %s, external dir %s",service.getName(), dir.getAbsolutePath()));
+ File[] jars = dir.listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ LOG.debug(String.format("Checking folder %s", name));
+ return name.endsWith(".jar");
+ }
+ });
+ for (File jar : jars) {
+ try {
+ URL url = jar.toURI().toURL();
+ urls.add(url);
+ LOG.info("Adding server action jar to classpath: {}", url);
+ }
+ catch (Exception e) {
+ LOG.error("Failed to add server action jar to classpath: {}", jar.getAbsolutePath(), e);
+ }
}
+ } else {
+ LOG.error(String.format("%s service server actions folder returned null", service));
+ }
+ }
+
+ ClassLoader classLoader = new URLClassLoader(urls.toArray(new URL[urls.size()]), ClassUtils.getDefaultClassLoader());
+ Class<?> actionClass = null;
+ try {
+ actionClass = ClassUtils.resolveClassName(classname, classLoader);
+ LOG.debug(String.format("Found external server action %s", classname));
+ } catch(IllegalArgumentException illegalArgumentException) {
+ LOG.error(String.format("Unable to find server action %s in external server action directories", classname), illegalArgumentException);
+ }
+
+ return actionClass;
+ }
+
+ /**
+ * Load server action classes defined in Ambari source code
+ * */
+ private Class<?> getServerActionClass(String classname) throws AmbariException{
+ Class<?> actionClass = null;
+ try {
+ actionClass = Class.forName(classname);
+ if (actionClass == null) {
+ LOG.warn(String.format("Unable to load server action class: %s from Ambari", classname));
}
} catch (ClassNotFoundException e) {
- throw new AmbariException("Unable to load server action class: " + classname, e);
+ LOG.error(String.format("Unable to load server action class: %s", classname), e);
}
+ return actionClass;
}
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/063ba36e/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java
index 119163e..7464e61 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceDirectory.java
@@ -93,6 +93,11 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory {
protected File checksDir;
/**
+ * server side action directory path
+ */
+ protected File serverActionsDir;
+
+ /**
* service metainfo file object representation
*/
private ServiceMetainfoXml metaInfoXml;
@@ -118,6 +123,11 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory {
protected static final String CHECKS_FOLDER_NAME = "checks";
/**
+ * Server actions directory name
+ */
+ protected static final String SERVER_ACTIONS_FOLDER_NAME = "server_actions";
+
+ /**
* service metainfo file name
*/
private static final String SERVICE_METAINFO_FILE_NAME = "metainfo.xml";
@@ -172,6 +182,15 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory {
}
/**
+ * Obtain the server side actions directory path.
+ *
+ * @return server side actions directory path
+ */
+ public File getServerActionsDir() {
+ return serverActionsDir;
+ }
+
+ /**
* Obtain the metrics file.
*
* @return metrics file
@@ -303,6 +322,7 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory {
calculatePackageDirectory(stack, service);
calculateUpgradesDirectory(stack, service);
calculateChecksDirectory(stack, service);
+ calculateServerActionsDirectory(stack, service);
}
/**
@@ -378,6 +398,15 @@ public abstract class ServiceDirectory extends StackDefinitionDirectory {
}
/**
+ * Sets the serverActionsDir if the dir exists and is not empty
+ * @param stack
+ * @param service
+ */
+ protected void calculateServerActionsDirectory(String stack, String service) {
+ serverActionsDir = resolveDirectory(SERVER_ACTIONS_FOLDER_NAME, stack, service);
+ }
+
+ /**
* Unmarshal the metainfo file into its object representation.
*
* @throws AmbariException if the metainfo file doesn't exist or
http://git-wip-us.apache.org/repos/asf/ambari/blob/063ba36e/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
index 3b3d52c..4eb3858 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/ServiceModule.java
@@ -150,6 +150,7 @@ public class ServiceModule extends BaseModule<ServiceModule, ServiceInfo> implem
serviceInfo.setServicePackageFolder(serviceDirectory.getPackageDir());
serviceInfo.setServiceUpgradesFolder(serviceDirectory.getUpgradesDir());
serviceInfo.setChecksFolder(serviceDirectory.getChecksDir());
+ serviceInfo.setServerActionsFolder(serviceDirectory.getServerActionsDir());
serviceInfo.setAdvisorFile(serviceDirectory.getAdvisorFile());
serviceInfo.setAdvisorName(serviceDirectory.getAdvisorName(serviceInfo.getName()));
@@ -270,6 +271,13 @@ public class ServiceModule extends BaseModule<ServiceModule, ServiceInfo> implem
}
/*
+ * Use parent's server actions if the current one does not have any.
+ */
+ if (serviceInfo.getServerActionsFolder() == null) {
+ serviceInfo.setServerActionsFolder(parent.getServerActionsFolder());
+ }
+
+ /*
* If current stack version does not specify the credential store information
* for the service, then use parent definition.
*/
http://git-wip-us.apache.org/repos/asf/ambari/blob/063ba36e/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java
index 8fe6583..f1c63bf 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java
@@ -305,6 +305,12 @@ public class ServiceInfo implements Validable{
@XmlTransient
private File checksFolder;
+ /**
+ * Stores the path to the server actions folder which contains server actions jars for the given service.
+ */
+ @XmlTransient
+ private File serverActionsFolder;
+
public boolean isDeleted() {
return isDeleted;
}
@@ -796,6 +802,14 @@ public class ServiceInfo implements Validable{
this.checksFolder = checksFolder;
}
+ public File getServerActionsFolder() {
+ return serverActionsFolder;
+ }
+
+ public void setServerActionsFolder(File serverActionsFolder) {
+ this.serverActionsFolder = serverActionsFolder;
+ }
+
/**
* Exposes (and initializes on first use) map of os-specific details.
* @return map of OS specific details keyed by family
http://git-wip-us.apache.org/repos/asf/ambari/blob/063ba36e/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java
index dbdd043..13c32cf 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/ServiceModuleTest.java
@@ -499,6 +499,36 @@ public class ServiceModuleTest {
}
@Test
+ public void testResolve_ServerActionDirectory() throws Exception {
+ File serverActions = new File("server_actions");
+
+ // check directory specified in child only
+ ServiceInfo info = new ServiceInfo();
+ ServiceInfo parentInfo = new ServiceInfo();
+ ServiceModule child = createServiceModule(info);
+ ServiceModule parent = createServiceModule(parentInfo);
+ child.getModuleInfo().setServerActionsFolder(serverActions);
+ resolveService(child, parent);
+ assertEquals(serverActions.getPath(), child.getModuleInfo().getServerActionsFolder().getPath());
+
+ // check directory specified in parent only
+ child = createServiceModule(info);
+ parent = createServiceModule(parentInfo);
+ parent.getModuleInfo().setServerActionsFolder(serverActions);
+ resolveService(child, parent);
+ assertEquals(serverActions.getPath(), child.getModuleInfo().getServerActionsFolder().getPath());
+
+ // check directory set in both
+ info.setServerActionsFolder(serverActions);
+ child = createServiceModule(info);
+ child.getModuleInfo().setServerActionsFolder(serverActions);
+ parent = createServiceModule(parentInfo);
+ parent.getModuleInfo().setServerActionsFolder(new File("other"));
+ resolveService(child, parent);
+ assertEquals(serverActions.getPath(), child.getModuleInfo().getServerActionsFolder().getPath());
+ }
+
+ @Test
public void testResolve_CustomCommands() throws Exception {
List<CustomCommandDefinition> customCommands = new ArrayList<>();
CustomCommandDefinition cmd1 = new CustomCommandDefinition();
http://git-wip-us.apache.org/repos/asf/ambari/blob/063ba36e/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java
index 6617b33..bf55e6f 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java
@@ -140,6 +140,9 @@ public class StackManagerExtensionTest {
File checks = oozie.getChecksFolder();
assertNotNull(checks);
assertTrue("Checks dir is " + checks.getPath(), checks.getPath().contains("extensions/EXT/0.1/services/OOZIE2/checks"));
+ File serverActions = oozie.getServerActionsFolder();
+ assertNotNull(serverActions);
+ assertTrue("Server actions dir is " + serverActions.getPath(), serverActions.getPath().contains("extensions/EXT/0.1/services/OOZIE2/server_actions"));
List<ThemeInfo> themes = oozie.getThemes();
assertNotNull(themes);
assertTrue("Number of themes is " + themes.size(), themes.size() == 1);
@@ -158,6 +161,9 @@ public class StackManagerExtensionTest {
checks = oozie.getChecksFolder();
assertNotNull(checks);
assertTrue("Checks dir is " + checks.getPath(), checks.getPath().contains("extensions/EXT/0.1/services/OOZIE2/checks"));
+ serverActions = oozie.getServerActionsFolder();
+ assertNotNull(serverActions);
+ assertTrue("Server actions dir is " + serverActions.getPath(), serverActions.getPath().contains("extensions/EXT/0.1/services/OOZIE2/server_actions"));
themes = oozie.getThemes();
assertNotNull(themes);
assertTrue("Number of themes is " + themes.size(), themes.size() == 0);