You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by tb...@apache.org on 2014/09/05 05:17:59 UTC
git commit: AMBARI-7158 - Views: Make view extraction a background
task.
Repository: ambari
Updated Branches:
refs/heads/trunk 5d487f9ac -> c86c78104
AMBARI-7158 - Views: Make view extraction a background task.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c86c7810
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c86c7810
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c86c7810
Branch: refs/heads/trunk
Commit: c86c781043c04940ccd488238fbd78b4b7ef15b9
Parents: 5d487f9
Author: tbeerbower <tb...@hortonworks.com>
Authored: Thu Sep 4 19:04:27 2014 -0400
Committer: tbeerbower <tb...@hortonworks.com>
Committed: Thu Sep 4 22:36:14 2014 -0400
----------------------------------------------------------------------
.../server/configuration/Configuration.java | 36 ++++
.../ambari/server/controller/AmbariServer.java | 10 +-
.../internal/ViewVersionResourceProvider.java | 22 +-
.../ambari/server/orm/entities/ViewEntity.java | 55 ++++-
.../apache/ambari/server/view/ViewRegistry.java | 200 ++++++++++++-------
.../server/configuration/ConfigurationTest.java | 19 +-
.../AmbariPrivilegeResourceProviderTest.java | 6 +-
.../ViewPrivilegeResourceProviderTest.java | 6 +-
.../server/orm/entities/ViewEntityTest.java | 30 ++-
.../ambari/server/view/ViewRegistryTest.java | 50 ++++-
.../org/apache/ambari/view/ViewDefinition.java | 27 +++
11 files changed, 351 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c86c7810/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index dc3a715..ae20d43 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -302,6 +302,12 @@ public class Configuration {
private static final String AGENT_THREADPOOL_SIZE_KEY = "agent.threadpool.size.max";
private static final int AGENT_THREADPOOL_SIZE_DEFAULT = 25;
+ private static final String VIEW_EXTRACTION_THREADPOOL_MAX_SIZE_KEY = "view.extraction.threadpool.size.max";
+ private static final int VIEW_EXTRACTION_THREADPOOL_MAX_SIZE_DEFAULT = 20;
+ private static final String VIEW_EXTRACTION_THREADPOOL_CORE_SIZE_KEY = "view.extraction.threadpool.size.core";
+ private static final int VIEW_EXTRACTION_THREADPOOL_CORE_SIZE_DEFAULT = 10;
+ private static final String VIEW_EXTRACTION_THREADPOOL_TIMEOUT_KEY = "view.extraction.threadpool.timeout";
+ private static final long VIEW_EXTRACTION_THREADPOOL_TIMEOUT_DEFAULT = 100000L;
private static final Logger LOG = LoggerFactory.getLogger(
Configuration.class);
@@ -1031,4 +1037,34 @@ public class Configuration {
return Integer.parseInt(properties.getProperty(
AGENT_THREADPOOL_SIZE_KEY, String.valueOf(AGENT_THREADPOOL_SIZE_DEFAULT)));
}
+
+ /**
+ * Get the view extraction thread pool max size.
+ *
+ * @return the view extraction thread pool max size
+ */
+ public int getViewExtractionThreadPoolMaxSize() {
+ return Integer.parseInt(properties.getProperty(
+ VIEW_EXTRACTION_THREADPOOL_MAX_SIZE_KEY, String.valueOf(VIEW_EXTRACTION_THREADPOOL_MAX_SIZE_DEFAULT)));
+ }
+
+ /**
+ * Get the view extraction thread pool core size.
+ *
+ * @return the view extraction thread pool core size
+ */
+ public int getViewExtractionThreadPoolCoreSize() {
+ return Integer.parseInt(properties.getProperty(
+ VIEW_EXTRACTION_THREADPOOL_CORE_SIZE_KEY, String.valueOf(VIEW_EXTRACTION_THREADPOOL_CORE_SIZE_DEFAULT)));
+ }
+
+ /**
+ * Get the view extraction thread pool timeout.
+ *
+ * @return the view extraction thread pool timeout
+ */
+ public long getViewExtractionThreadPoolTimeout() {
+ return Long.parseLong(properties.getProperty(
+ VIEW_EXTRACTION_THREADPOOL_TIMEOUT_KEY, String.valueOf(VIEW_EXTRACTION_THREADPOOL_TIMEOUT_DEFAULT)));
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c86c7810/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
index 955bf0a..d83138a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java
@@ -70,7 +70,6 @@ import org.apache.ambari.server.orm.dao.PrivilegeDAO;
import org.apache.ambari.server.orm.dao.ResourceDAO;
import org.apache.ambari.server.orm.dao.UserDAO;
import org.apache.ambari.server.orm.entities.MetainfoEntity;
-import org.apache.ambari.server.orm.entities.ViewInstanceEntity;
import org.apache.ambari.server.resources.ResourceManager;
import org.apache.ambari.server.resources.api.rest.GetResource;
import org.apache.ambari.server.scheduler.ExecutionScheduleManager;
@@ -90,7 +89,6 @@ import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.utils.StageUtils;
import org.apache.ambari.server.utils.VersionUtils;
import org.apache.ambari.server.view.ViewRegistry;
-import org.apache.ambari.view.SystemException;
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.nio.SelectChannelConnector;
@@ -307,13 +305,7 @@ public class AmbariServer {
root.addServlet(sh, "/api/v1/*");
sh.setInitOrder(2);
- try {
- for (ViewInstanceEntity entity : viewRegistry.readViewArchives(configs)){
- handlerList.addViewInstance(entity);
- }
- } catch (SystemException e) {
- LOG.error("Caught exception deploying views.", e);
- }
+ viewRegistry.readViewArchives();
handlerList.addHandler(root);
http://git-wip-us.apache.org/repos/asf/ambari/blob/c86c7810/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ViewVersionResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ViewVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ViewVersionResourceProvider.java
index d13785f..e2138be 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ViewVersionResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ViewVersionResourceProvider.java
@@ -44,14 +44,16 @@ public class ViewVersionResourceProvider extends AbstractResourceProvider {
/**
* View property id constants.
*/
- public static final String VIEW_NAME_PROPERTY_ID = "ViewVersionInfo/view_name";
- public static final String VIEW_VERSION_PROPERTY_ID = "ViewVersionInfo/version";
- public static final String LABEL_PROPERTY_ID = "ViewVersionInfo/label";
- public static final String DESCRIPTION_PROPERTY_ID = "ViewVersionInfo/description";
- public static final String VERSION_PROPERTY_ID = "ViewVersionInfo/version";
- public static final String PARAMETERS_PROPERTY_ID = "ViewVersionInfo/parameters";
- public static final String ARCHIVE_PROPERTY_ID = "ViewVersionInfo/archive";
- public static final String MASKER_CLASS_PROPERTY_ID = "ViewVersionInfo/masker_class";
+ public static final String VIEW_NAME_PROPERTY_ID = "ViewVersionInfo/view_name";
+ public static final String VIEW_VERSION_PROPERTY_ID = "ViewVersionInfo/version";
+ public static final String LABEL_PROPERTY_ID = "ViewVersionInfo/label";
+ public static final String DESCRIPTION_PROPERTY_ID = "ViewVersionInfo/description";
+ public static final String VERSION_PROPERTY_ID = "ViewVersionInfo/version";
+ public static final String PARAMETERS_PROPERTY_ID = "ViewVersionInfo/parameters";
+ public static final String ARCHIVE_PROPERTY_ID = "ViewVersionInfo/archive";
+ public static final String MASKER_CLASS_PROPERTY_ID = "ViewVersionInfo/masker_class";
+ public static final String VIEW_STATUS_PROPERTY_ID = "ViewVersionInfo/status";
+ public static final String VIEW_STATUS_DETAIL_PROPERTY_ID = "ViewVersionInfo/status_detail";
/**
* The key property ids for a view resource.
@@ -75,6 +77,8 @@ public class ViewVersionResourceProvider extends AbstractResourceProvider {
propertyIds.add(PARAMETERS_PROPERTY_ID);
propertyIds.add(ARCHIVE_PROPERTY_ID);
propertyIds.add(MASKER_CLASS_PROPERTY_ID);
+ propertyIds.add(VIEW_STATUS_PROPERTY_ID);
+ propertyIds.add(VIEW_STATUS_DETAIL_PROPERTY_ID);
}
@@ -129,6 +133,8 @@ public class ViewVersionResourceProvider extends AbstractResourceProvider {
viewDefinition.getConfiguration().getParameters(), requestedIds);
setResourceProperty(resource, ARCHIVE_PROPERTY_ID, viewDefinition.getArchive(), requestedIds);
setResourceProperty(resource, MASKER_CLASS_PROPERTY_ID, viewDefinition.getMask(), requestedIds);
+ setResourceProperty(resource, VIEW_STATUS_PROPERTY_ID, viewDefinition.getStatus().toString(), requestedIds);
+ setResourceProperty(resource, VIEW_STATUS_DETAIL_PROPERTY_ID, viewDefinition.getStatusDetail(), requestedIds);
resources.add(resource);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c86c7810/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ViewEntity.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ViewEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ViewEntity.java
index 998d2c1..aec9ab9 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ViewEntity.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ViewEntity.java
@@ -171,7 +171,7 @@ public class ViewEntity implements ViewDefinition {
* The classloader used to load the view.
*/
@Transient
- private final ClassLoader classLoader;
+ private ClassLoader classLoader = null;
/**
* The mapping of resource type to resource provider.
@@ -203,6 +203,18 @@ public class ViewEntity implements ViewDefinition {
@Transient
private View view = null;
+ /**
+ * The view status.
+ */
+ @Transient
+ private ViewStatus status = ViewStatus.PENDING;
+
+ /**
+ * The view status detail.
+ */
+ @Transient
+ private String statusDetail;
+
// ----- Constructors ------------------------------------------------------
@@ -212,7 +224,6 @@ public class ViewEntity implements ViewDefinition {
public ViewEntity() {
this.configuration = null;
this.ambariConfiguration = null;
- this.classLoader = null;
this.archive = null;
this.externalResourceType = null;
}
@@ -222,14 +233,12 @@ public class ViewEntity implements ViewDefinition {
*
* @param configuration the view configuration
* @param ambariConfiguration the Ambari configuration
- * @param classLoader the class loader
* @param archivePath the path of the view archive
*/
public ViewEntity(ViewConfig configuration, Configuration ambariConfiguration,
- ClassLoader classLoader, String archivePath) {
+ String archivePath) {
this.configuration = configuration;
this.ambariConfiguration = ambariConfiguration;
- this.classLoader = classLoader;
this.archive = archivePath;
String version = configuration.getVersion();
@@ -270,6 +279,16 @@ public class ViewEntity implements ViewDefinition {
return version;
}
+ @Override
+ public ViewStatus getStatus() {
+ return status;
+ }
+
+ @Override
+ public String getStatusDetail() {
+ return statusDetail;
+ }
+
// ----- ViewEntity --------------------------------------------------------
@@ -563,6 +582,15 @@ public class ViewEntity implements ViewDefinition {
}
/**
+ * Set the class loader.
+ *
+ * @param classLoader the class loader
+ */
+ public void setClassLoader(ClassLoader classLoader) {
+ this.classLoader = classLoader;
+ }
+
+ /**
* Add a resource provider for the given type.
*
* @param type the resource type
@@ -703,8 +731,23 @@ public class ViewEntity implements ViewDefinition {
this.resourceType = resourceType;
}
+ /**
+ * Set the status of the view.
+ *
+ * @param status the view status
+ */
+ public void setStatus(ViewStatus status) {
+ this.status = status;
+ }
-// ----- helper methods ----------------------------------------------------
+ /**
+ * Set the status detail for the view.
+ *
+ * @param statusDetail the status detail
+ */
+ public void setStatusDetail(String statusDetail) {
+ this.statusDetail = statusDetail;
+ }
/**
* Get the internal view name from the given common name and version.
http://git-wip-us.apache.org/repos/asf/ambari/blob/c86c7810/ambari-server/src/main/java/org/apache/ambari/server/view/ViewRegistry.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/view/ViewRegistry.java b/ambari-server/src/main/java/org/apache/ambari/server/view/ViewRegistry.java
index 36c9081..859ea96 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/view/ViewRegistry.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/view/ViewRegistry.java
@@ -38,6 +38,10 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
@@ -118,6 +122,11 @@ public class ViewRegistry {
private static final String EXTRACTED_ARCHIVES_DIR = "work";
/**
+ * Thread pool
+ */
+ private static ExecutorService executorService;
+
+ /**
* Mapping of view names to view definitions.
*/
private Map<String, ViewEntity> viewDefinitions = new HashMap<String, ViewEntity>();
@@ -262,10 +271,6 @@ public class ViewRegistry {
* @param definition the definition
*/
public void addDefinition(ViewEntity definition) {
- View view = definition.getView();
- if (view != null) {
- view.onDeploy(definition);
- }
viewDefinitions.put(definition.getName(), definition);
}
@@ -391,80 +396,56 @@ public class ViewRegistry {
}
/**
- * Read the view archives.
- *
- * @param configuration Ambari configuration
- *
- * @return the set of view instance definitions read from the archives
- *
- * @throws SystemException if the view archives can not be successfully read
+ * Asynchronously read the view archives.
*/
- public Set<ViewInstanceEntity> readViewArchives(Configuration configuration)
- throws SystemException {
-
- try {
- File viewDir = configuration.getViewsDir();
-
- Set<ViewInstanceEntity> allInstanceDefinitions = new HashSet<ViewInstanceEntity>();
-
- String extractedArchivesPath = viewDir.getAbsolutePath() +
- File.separator + EXTRACTED_ARCHIVES_DIR;
+ public void readViewArchives() {
- if (ensureExtractedArchiveDirectory(extractedArchivesPath)) {
- File[] files = viewDir.listFiles();
+ final ExecutorService executorService = getExecutorService(configuration);
- if (files != null) {
- for (File archiveFile : files) {
- if (!archiveFile.isDirectory()) {
- try {
- ViewConfig viewConfig = helper.getViewConfigFromArchive(archiveFile);
+ // submit a task to manage the extraction tasks
+ executorService.submit(new Runnable() {
+ @Override
+ public void run() {
- String viewName = ViewEntity.getViewName(viewConfig.getName(), viewConfig.getVersion());
- String archivePath = extractedArchivesPath + File.separator + viewName;
+ try {
+ File viewDir = configuration.getViewsDir();
- // extract the archive and get the class loader
- ClassLoader cl = extractViewArchive(archiveFile, helper.getFile(archivePath));
+ String extractedArchivesPath = viewDir.getAbsolutePath() +
+ File.separator + EXTRACTED_ARCHIVES_DIR;
- viewConfig = helper.getViewConfigFromExtractedArchive(archivePath);
+ if (ensureExtractedArchiveDirectory(extractedArchivesPath)) {
+ File[] files = viewDir.listFiles();
- ViewEntity viewDefinition = createViewDefinition(viewConfig, configuration, cl, archivePath);
+ if (files != null) {
+ for (final File archiveFile : files) {
+ if (!archiveFile.isDirectory()) {
- Set<ViewInstanceEntity> instanceDefinitions = new HashSet<ViewInstanceEntity>();
+ final ViewConfig viewConfig = helper.getViewConfigFromArchive(archiveFile);
- for (InstanceConfig instanceConfig : viewConfig.getInstances()) {
- try {
- ViewInstanceEntity instanceEntity = createViewInstanceDefinition(viewConfig, viewDefinition, instanceConfig);
- instanceEntity.setXmlDriven(true);
- instanceDefinitions.add(instanceEntity);
- } catch (Exception e) {
- LOG.error("Caught exception adding view instance for view " +
- viewDefinition.getViewName(), e);
- }
- }
- // ensure that the view entity matches the db
- syncView(viewDefinition, instanceDefinitions);
+ String commonName = viewConfig.getName();
+ String version = viewConfig.getVersion();
+ String viewName = ViewEntity.getViewName(commonName, version);
- // update the registry with the view
- addDefinition(viewDefinition);
+ final String archivePath = extractedArchivesPath + File.separator + viewName;
+ final ViewEntity viewDefinition = new ViewEntity(viewConfig, configuration, archivePath);
- // update the registry with the view instances
- for (ViewInstanceEntity instanceEntity : instanceDefinitions) {
- addInstanceDefinition(viewDefinition, instanceEntity);
+ // submit a new task for each archive being read
+ executorService.submit(new Runnable() {
+ @Override
+ public void run() {
+ readViewArchive(viewDefinition, archiveFile, archivePath, viewConfig);
+ }
+ });
}
-
- allInstanceDefinitions.addAll(instanceDefinitions);
- } catch (Exception e) {
- LOG.error("Caught exception loading view from " + archiveFile.getAbsolutePath(), e);
}
+ removeUndeployedViews();
}
}
- removeUndeployedViews();
+ } catch (Exception e) {
+ LOG.error("Caught exception reading view archives.", e);
}
}
- return allInstanceDefinitions;
- } catch (Exception e) {
- throw new SystemException("Caught exception reading view archives.", e);
- }
+ });
}
/**
@@ -786,12 +767,12 @@ public class ViewRegistry {
return viewDefinitions.get(viewName);
}
- // create a new view definition
- protected ViewEntity createViewDefinition(ViewConfig viewConfig, Configuration ambariConfig,
- ClassLoader cl, String archivePath)
+ // setup the given view definition
+ protected ViewEntity setupViewDefinition(ViewEntity viewDefinition, ViewConfig viewConfig,
+ ClassLoader cl)
throws ClassNotFoundException, IntrospectionException {
- ViewEntity viewDefinition = new ViewEntity(viewConfig, ambariConfig, cl, archivePath);
+ viewDefinition.setClassLoader(cl);
List<ParameterConfig> parameterConfigurations = viewConfig.getParameters();
@@ -1145,7 +1126,7 @@ public class ViewRegistry {
}
// extract the given view archive to the given archive directory
- private ClassLoader extractViewArchive(File viewArchive, File archiveDir)
+ private ClassLoader extractViewArchive(ViewEntity viewDefinition, File viewArchive, File archiveDir)
throws IOException {
// Skip if the archive has already been extracted
@@ -1153,13 +1134,17 @@ public class ViewRegistry {
String archivePath = archiveDir.getAbsolutePath();
- LOG.info("Creating archive folder " + archivePath + ".");
+ String msg = "Creating archive folder " + archivePath + ".";
+ LOG.info(msg);
+ setViewStatus(viewDefinition, ViewDefinition.ViewStatus.LOADING, msg);
if (archiveDir.mkdir()) {
JarFile viewJarFile = helper.getJarFile(viewArchive);
Enumeration enumeration = viewJarFile.entries();
- LOG.info("Extracting files from " + viewArchive.getName() + ":");
+ msg = "Extracting files from " + viewArchive.getName() + ":";
+ LOG.info(msg);
+ setViewStatus(viewDefinition, ViewDefinition.ViewStatus.LOADING, msg);
while (enumeration.hasMoreElements()) {
JarEntry jarEntry = (JarEntry) enumeration.nextElement();
@@ -1268,6 +1253,85 @@ public class ViewRegistry {
return false;
}
+ // fire the onDeploy event.
+ protected void onDeploy(ViewEntity definition) {
+ View view = definition.getView();
+ if (view != null) {
+ view.onDeploy(definition);
+ }
+ }
+
+ // read a view archive and return the set of new view instances
+ private void readViewArchive(ViewEntity viewDefinition,
+ File archiveFile,
+ String archivePath,
+ ViewConfig viewConfig) {
+
+ setViewStatus(viewDefinition, ViewEntity.ViewStatus.LOADING, "Loading " + archivePath + ".");
+
+ try {
+ // update the registry with the view
+ addDefinition(viewDefinition);
+
+ // extract the archive and get the class loader
+ ClassLoader cl = extractViewArchive(viewDefinition, archiveFile, helper.getFile(archivePath));
+
+ viewConfig = helper.getViewConfigFromExtractedArchive(archivePath);
+
+ setupViewDefinition(viewDefinition, viewConfig, cl);
+
+ Set<ViewInstanceEntity> instanceDefinitions = new HashSet<ViewInstanceEntity>();
+
+ for (InstanceConfig instanceConfig : viewConfig.getInstances()) {
+ ViewInstanceEntity instanceEntity = createViewInstanceDefinition(viewConfig, viewDefinition, instanceConfig);
+ instanceEntity.setXmlDriven(true);
+ instanceDefinitions.add(instanceEntity);
+ }
+ // ensure that the view entity matches the db
+ syncView(viewDefinition, instanceDefinitions);
+
+ onDeploy(viewDefinition);
+
+ // update the registry with the view instances
+ for (ViewInstanceEntity instanceEntity : instanceDefinitions) {
+ addInstanceDefinition(viewDefinition, instanceEntity);
+ handlerList.addViewInstance(instanceEntity);
+ }
+ setViewStatus(viewDefinition, ViewEntity.ViewStatus.LOADED, "Loaded " + archivePath + ".");
+
+ } catch (Exception e) {
+ String msg = "Caught exception loading view " + viewDefinition.getViewName() + " : " + e.getMessage();
+
+ setViewStatus(viewDefinition, ViewEntity.ViewStatus.ERROR, msg);
+ LOG.error(msg, e);
+ }
+ }
+
+ // set the status of the given view.
+ private void setViewStatus(ViewEntity viewDefinition, ViewEntity.ViewStatus status, String statusDetail) {
+ viewDefinition.setStatus(status);
+ viewDefinition.setStatusDetail(statusDetail);
+ }
+
+ // Get the view extraction thread pool
+ private static synchronized ExecutorService getExecutorService(Configuration configuration) {
+ if (executorService == null) {
+ LinkedBlockingQueue<Runnable> queue = new LinkedBlockingQueue<Runnable>();
+
+ ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(
+ configuration.getViewExtractionThreadPoolCoreSize(),
+ configuration.getViewExtractionThreadPoolMaxSize(),
+ configuration.getViewExtractionThreadPoolTimeout(),
+ TimeUnit.MILLISECONDS,
+ queue);
+
+ threadPoolExecutor.allowCoreThreadTimeOut(true);
+ executorService = threadPoolExecutor;
+ }
+ return executorService;
+ }
+
+
// ----- inner class : ViewRegistryHelper ----------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c86c7810/ambari-server/src/test/java/org/apache/ambari/server/configuration/ConfigurationTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/configuration/ConfigurationTest.java b/ambari-server/src/test/java/org/apache/ambari/server/configuration/ConfigurationTest.java
index 79d44b0..54154b1 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/configuration/ConfigurationTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/configuration/ConfigurationTest.java
@@ -268,15 +268,26 @@ public class ConfigurationTest {
Assert.assertEquals(25, conf.getClientThreadPoolSize());
Assert.assertEquals(25, conf.getAgentThreadPoolSize());
-
+
+ Assert.assertEquals(10, conf.getViewExtractionThreadPoolCoreSize());
+ Assert.assertEquals(20, conf.getViewExtractionThreadPoolMaxSize());
+ Assert.assertEquals(100000L, conf.getViewExtractionThreadPoolTimeout());
+
ambariProperties = new Properties();
ambariProperties.setProperty("client.threadpool.size.max", "4");
ambariProperties.setProperty("agent.threadpool.size.max", "82");
+
+ ambariProperties.setProperty("view.extraction.threadpool.size.core", "83");
+ ambariProperties.setProperty("view.extraction.threadpool.size.max", "56");
+ ambariProperties.setProperty("view.extraction.threadpool.timeout", "6000");
+
conf = new Configuration(ambariProperties);
Assert.assertEquals(4, conf.getClientThreadPoolSize());
Assert.assertEquals(82, conf.getAgentThreadPoolSize());
-
- }
-
+
+ Assert.assertEquals(83, conf.getViewExtractionThreadPoolCoreSize());
+ Assert.assertEquals(56, conf.getViewExtractionThreadPoolMaxSize());
+ Assert.assertEquals(6000L, conf.getViewExtractionThreadPoolTimeout());
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c86c7810/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/AmbariPrivilegeResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/AmbariPrivilegeResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/AmbariPrivilegeResourceProviderTest.java
index 243c98c..fbeb0cc 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/AmbariPrivilegeResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/AmbariPrivilegeResourceProviderTest.java
@@ -62,6 +62,7 @@ import org.apache.ambari.server.orm.entities.UserEntity;
import org.apache.ambari.server.orm.entities.ViewEntity;
import org.apache.ambari.server.orm.entities.ViewInstanceEntity;
import org.apache.ambari.server.security.SecurityHelper;
+import org.apache.ambari.server.view.ViewInstanceHandlerList;
import org.apache.ambari.server.view.ViewRegistry;
import org.apache.ambari.server.view.ViewRegistryTest;
import org.easymock.EasyMock;
@@ -86,6 +87,7 @@ public class AmbariPrivilegeResourceProviderTest {
private static final MemberDAO memberDAO = createNiceMock(MemberDAO.class);
private static final ResourceTypeDAO resourceTypeDAO = createNiceMock(ResourceTypeDAO.class);
private static final SecurityHelper securityHelper = createNiceMock(SecurityHelper.class);
+ private static final ViewInstanceHandlerList handlerList = createNiceMock(ViewInstanceHandlerList.class);
@BeforeClass
public static void initClass() {
@@ -96,8 +98,8 @@ public class AmbariPrivilegeResourceProviderTest {
@Before
public void resetGlobalMocks() {
ViewRegistry.initInstance(ViewRegistryTest.getRegistry(viewDAO, viewInstanceDAO, userDAO,
- memberDAO, privilegeDAO, resourceDAO, resourceTypeDAO, securityHelper));
- reset(privilegeDAO, userDAO, groupDAO, principalDAO, permissionDAO, resourceDAO, clusterDAO);
+ memberDAO, privilegeDAO, resourceDAO, resourceTypeDAO, securityHelper, handlerList));
+ reset(privilegeDAO, userDAO, groupDAO, principalDAO, permissionDAO, resourceDAO, clusterDAO, handlerList);
}
@Test
http://git-wip-us.apache.org/repos/asf/ambari/blob/c86c7810/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ViewPrivilegeResourceProviderTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ViewPrivilegeResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ViewPrivilegeResourceProviderTest.java
index a092735..80f7751 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ViewPrivilegeResourceProviderTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ViewPrivilegeResourceProviderTest.java
@@ -42,6 +42,7 @@ import org.apache.ambari.server.orm.entities.ViewEntityTest;
import org.apache.ambari.server.orm.entities.ViewInstanceEntity;
import org.apache.ambari.server.orm.entities.ViewInstanceEntityTest;
import org.apache.ambari.server.security.SecurityHelper;
+import org.apache.ambari.server.view.ViewInstanceHandlerList;
import org.apache.ambari.server.view.ViewRegistry;
import org.apache.ambari.server.view.ViewRegistryTest;
import org.junit.Assert;
@@ -77,6 +78,7 @@ public class ViewPrivilegeResourceProviderTest {
private static final MemberDAO memberDAO = createNiceMock(MemberDAO.class);
private static final ResourceTypeDAO resourceTypeDAO = createNiceMock(ResourceTypeDAO.class);
private static final SecurityHelper securityHelper = createNiceMock(SecurityHelper.class);
+ private static final ViewInstanceHandlerList handlerList = createNiceMock(ViewInstanceHandlerList.class);
@BeforeClass
public static void initClass() {
@@ -87,8 +89,8 @@ public class ViewPrivilegeResourceProviderTest {
public void resetGlobalMocks() {
ViewRegistry.initInstance(ViewRegistryTest.getRegistry(viewDAO, viewInstanceDAO, userDAO,
- memberDAO, privilegeDAO, resourceDAO, resourceTypeDAO, securityHelper));
- reset(privilegeDAO, userDAO, groupDAO, principalDAO, permissionDAO, resourceDAO);
+ memberDAO, privilegeDAO, resourceDAO, resourceTypeDAO, securityHelper, handlerList));
+ reset(privilegeDAO, userDAO, groupDAO, principalDAO, permissionDAO, resourceDAO, handlerList);
}
@Test
http://git-wip-us.apache.org/repos/asf/ambari/blob/c86c7810/ambari-server/src/test/java/org/apache/ambari/server/orm/entities/ViewEntityTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/orm/entities/ViewEntityTest.java b/ambari-server/src/test/java/org/apache/ambari/server/orm/entities/ViewEntityTest.java
index f9719e6..115c2bf 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/orm/entities/ViewEntityTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/orm/entities/ViewEntityTest.java
@@ -25,6 +25,7 @@ import org.apache.ambari.server.view.configuration.ResourceConfig;
import org.apache.ambari.server.view.configuration.ResourceConfigTest;
import org.apache.ambari.server.view.configuration.ViewConfig;
import org.apache.ambari.server.view.configuration.ViewConfigTest;
+import org.apache.ambari.view.ViewDefinition;
import org.junit.Assert;
import org.junit.Test;
@@ -55,7 +56,9 @@ public class ViewEntityTest {
properties.put("p3", "v3");
Configuration ambariConfig = new Configuration(properties);
- ViewEntity viewEntity = new ViewEntity(viewConfig, ambariConfig, ViewEntityTest.class.getClassLoader(), "view.jar");
+ ViewEntity viewEntity = new ViewEntity(viewConfig, ambariConfig, "view.jar");
+
+ viewEntity.setClassLoader(ViewEntityTest.class.getClassLoader());
ResourceTypeEntity resourceTypeEntity = new ResourceTypeEntity();
resourceTypeEntity.setId(10);
@@ -238,4 +241,29 @@ public class ViewEntityTest {
Assert.assertEquals("v2", configuration.getProperty("p2"));
Assert.assertEquals("v3", configuration.getProperty("p3"));
}
+
+ @Test
+ public void testGetSetStatus() throws Exception {
+ ViewEntity viewDefinition = getViewEntity();
+
+ viewDefinition.setStatus(ViewDefinition.ViewStatus.PENDING);
+ Assert.assertEquals(ViewDefinition.ViewStatus.PENDING, viewDefinition.getStatus());
+
+ viewDefinition.setStatus(ViewDefinition.ViewStatus.LOADING);
+ Assert.assertEquals(ViewDefinition.ViewStatus.LOADING, viewDefinition.getStatus());
+
+ viewDefinition.setStatus(ViewDefinition.ViewStatus.LOADED);
+ Assert.assertEquals(ViewDefinition.ViewStatus.LOADED, viewDefinition.getStatus());
+
+ viewDefinition.setStatus(ViewDefinition.ViewStatus.ERROR);
+ Assert.assertEquals(ViewDefinition.ViewStatus.ERROR, viewDefinition.getStatus());
+ }
+
+ @Test
+ public void testGetSetStatusDetail() throws Exception {
+ ViewEntity viewDefinition = getViewEntity();
+
+ viewDefinition.setStatusDetail("status detail");
+ Assert.assertEquals("status detail", viewDefinition.getStatusDetail());
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c86c7810/ambari-server/src/test/java/org/apache/ambari/server/view/ViewRegistryTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/view/ViewRegistryTest.java b/ambari-server/src/test/java/org/apache/ambari/server/view/ViewRegistryTest.java
index dea9b3b..217f987 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/view/ViewRegistryTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/view/ViewRegistryTest.java
@@ -76,6 +76,7 @@ import org.apache.ambari.server.view.configuration.ViewConfig;
import org.apache.ambari.server.view.configuration.ViewConfigTest;
import org.apache.ambari.server.view.events.EventImpl;
import org.apache.ambari.server.view.events.EventImplTest;
+import org.apache.ambari.view.ViewDefinition;
import org.apache.ambari.view.events.Event;
import org.apache.ambari.view.events.Listener;
import org.easymock.EasyMock;
@@ -164,7 +165,6 @@ public class ViewRegistryTest {
public void testReadViewArchives() throws Exception {
ViewRegistry registry = getRegistry();
- Configuration configuration = createNiceMock(Configuration.class);
File viewDir = createNiceMock(File.class);
File extractedArchiveDir = createNiceMock(File.class);
File viewArchive = createNiceMock(File.class);
@@ -219,6 +219,10 @@ public class ViewRegistryTest {
expect(configuration.getViewsDir()).andReturn(viewDir);
expect(viewDir.getAbsolutePath()).andReturn("/var/lib/ambari-server/resources/views");
+ expect(configuration.getViewExtractionThreadPoolCoreSize()).andReturn(2).anyTimes();
+ expect(configuration.getViewExtractionThreadPoolMaxSize()).andReturn(3).anyTimes();
+ expect(configuration.getViewExtractionThreadPoolTimeout()).andReturn(10000L).anyTimes();
+
expect(viewDir.listFiles()).andReturn(new File[]{viewArchive});
expect(viewArchive.isDirectory()).andReturn(false);
@@ -270,9 +274,21 @@ public class ViewRegistryTest {
registry.setHelper(new TestViewRegistryHelper(viewConfigs, files, outputStreams, jarFiles));
- Set<ViewInstanceEntity> instanceEntities = registry.readViewArchives(configuration);
+ registry.readViewArchives();
+
+ ViewEntity view = null;
+
+ // Wait for the view load to complete.
+ long timeout = System.currentTimeMillis() + 10000L;
+ while ((view == null || !view.getStatus().equals(ViewDefinition.ViewStatus.LOADED))&&
+ System.currentTimeMillis() < timeout) {
+ view = registry.getDefinition("MY_VIEW", "1.0.0");
+ }
+
+ Assert.assertNotNull(view);
+ Assert.assertEquals(ViewDefinition.ViewStatus.LOADED, view.getStatus());
- Assert.assertEquals(2, instanceEntities.size());
+ Assert.assertEquals(2, registry.getInstanceDefinitions(view).size());
// verify mocks
verify(configuration, viewDir, extractedArchiveDir, viewArchive, archiveDir, entryFile, classesDir,
@@ -283,7 +299,6 @@ public class ViewRegistryTest {
public void testReadViewArchives_exception() throws Exception {
ViewRegistry registry = getRegistry();
- Configuration configuration = createNiceMock(Configuration.class);
File viewDir = createNiceMock(File.class);
File extractedArchiveDir = createNiceMock(File.class);
File viewArchive = createNiceMock(File.class);
@@ -388,9 +403,19 @@ public class ViewRegistryTest {
registry.setHelper(new TestViewRegistryHelper(viewConfigs, files, outputStreams, jarFiles));
- Set<ViewInstanceEntity> instanceEntities = registry.readViewArchives(configuration);
+ registry.readViewArchives();
- Assert.assertEquals(0, instanceEntities.size());
+ ViewEntity view = null;
+
+ // Wait for the view load to complete.
+ long timeout = System.currentTimeMillis() + 10000L;
+ while ((view == null || !view.getStatus().equals(ViewDefinition.ViewStatus.ERROR))&&
+ System.currentTimeMillis() < timeout) {
+ view = registry.getDefinition("MY_VIEW", "1.0.0");
+ }
+
+ Assert.assertNotNull(view);
+ Assert.assertEquals(ViewDefinition.ViewStatus.ERROR, view.getStatus());
// verify mocks
verify(configuration, viewDir, extractedArchiveDir, viewArchive, archiveDir, entryFile, classesDir,
@@ -885,10 +910,10 @@ public class ViewRegistryTest {
private static ViewRegistry getRegistry() {
ViewRegistry instance = getRegistry(viewDAO, viewInstanceDAO,
userDAO, memberDAO, privilegeDAO,
- resourceDAO, resourceTypeDAO, securityHelper);
+ resourceDAO, resourceTypeDAO, securityHelper, handlerList);
reset(viewDAO, resourceDAO, viewInstanceDAO, userDAO, memberDAO,
- privilegeDAO, resourceTypeDAO, securityHelper, configuration);
+ privilegeDAO, resourceTypeDAO, securityHelper, configuration, handlerList);
return instance;
}
@@ -896,7 +921,8 @@ public class ViewRegistryTest {
public static ViewRegistry getRegistry(ViewDAO viewDAO, ViewInstanceDAO viewInstanceDAO,
UserDAO userDAO, MemberDAO memberDAO,
PrivilegeDAO privilegeDAO, ResourceDAO resourceDAO,
- ResourceTypeDAO resourceTypeDAO, SecurityHelper securityHelper ) {
+ ResourceTypeDAO resourceTypeDAO, SecurityHelper securityHelper,
+ ViewInstanceHandlerList handlerList) {
ViewRegistry instance = new ViewRegistry();
@@ -918,7 +944,11 @@ public class ViewRegistryTest {
ClassLoader cl, String archivePath) throws Exception{
ViewRegistry registry = getRegistry();
- return registry.createViewDefinition(viewConfig, ambariConfig, cl, archivePath);
+ ViewEntity viewDefinition = new ViewEntity(viewConfig, ambariConfig, archivePath);
+
+ registry.setupViewDefinition(viewDefinition, viewConfig, cl);
+
+ return viewDefinition;
}
public static ViewInstanceEntity getViewInstanceEntity(ViewEntity viewDefinition, InstanceConfig instanceConfig) throws Exception {
http://git-wip-us.apache.org/repos/asf/ambari/blob/c86c7810/ambari-views/src/main/java/org/apache/ambari/view/ViewDefinition.java
----------------------------------------------------------------------
diff --git a/ambari-views/src/main/java/org/apache/ambari/view/ViewDefinition.java b/ambari-views/src/main/java/org/apache/ambari/view/ViewDefinition.java
index b8e7444..18b86ff 100644
--- a/ambari-views/src/main/java/org/apache/ambari/view/ViewDefinition.java
+++ b/ambari-views/src/main/java/org/apache/ambari/view/ViewDefinition.java
@@ -57,4 +57,31 @@ public interface ViewDefinition {
* @return the mask class name.
*/
public String getMask();
+
+ /**
+ * Get the view status.
+ *
+ * @return the view status
+ */
+ public ViewStatus getStatus();
+
+ /**
+ * Get the view status detail.
+ *
+ * @return the view status detail
+ */
+ public String getStatusDetail();
+
+
+ // ----- ViewStatus enum ---------------------------------------------------
+
+ /**
+ * View status
+ */
+ public enum ViewStatus {
+ PENDING, // view has been created but not loaded from the archive
+ LOADING, // view is in the process of being loaded from the archive
+ LOADED, // view is completely loaded and ready to use
+ ERROR // an error occurred loading the view
+ }
}