You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by ki...@apache.org on 2013/09/20 20:30:17 UTC
[08/15] Adding Helix-task-framework and Yarn integration modules
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/BootUtils.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/BootUtils.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/BootUtils.java
new file mode 100644
index 0000000..fbbfa14
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/BootUtils.java
@@ -0,0 +1,127 @@
+package org.apache.helix.metamanager.bootstrap;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.helix.metamanager.container.ContainerProcessProperties;
+import org.apache.log4j.Logger;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+public class BootUtils {
+
+ public static final String CLASS_PROPERTY = "class";
+ static final Logger log = Logger.getLogger(BootUtils.class);
+
+ public static boolean hasNamespace(Properties properties, String namespace) {
+ String prefix = namespace + ".";
+ for (String key : properties.stringPropertyNames()) {
+ if (key.startsWith(prefix))
+ return true;
+ }
+ return false;
+ }
+
+ public static Set<String> getNamespaces(Properties properties) {
+ Pattern pattern = Pattern.compile("^([^\\.\\=]+)");
+
+ Set<String> namespaces = Sets.newHashSet();
+
+ for (Map.Entry<Object, Object> rawEntry : properties.entrySet()) {
+ String key = (String) rawEntry.getKey();
+
+ Matcher matcher = pattern.matcher(key);
+ if(matcher.find()) {
+ namespaces.add(matcher.group(1));
+ }
+ }
+
+ return namespaces;
+ }
+
+ public static Properties getNamespace(Properties source, String namespace) {
+ Properties dest = new Properties();
+ String prefix = namespace + ".";
+
+ for (Map.Entry<Object, Object> rawEntry : source.entrySet()) {
+ String key = (String) rawEntry.getKey();
+ String value = (String) rawEntry.getValue();
+
+ if (key.startsWith(prefix)) {
+ String newKey = key.substring(prefix.length());
+ dest.put(newKey, value);
+ }
+ }
+
+ return dest;
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <T> T createInstance(Properties properties) throws Exception {
+ String className = properties.getProperty(CLASS_PROPERTY);
+
+ Class<?> containerClass = Class.forName(className);
+
+ try {
+ log.debug(String.format("checking for properties constructor in class '%s'", className));
+ return (T) containerClass.getConstructor(ContainerProcessProperties.class).newInstance(properties);
+ } catch (Exception e) {
+ log.debug("no properties constructor found");
+ }
+
+ try {
+ log.debug(String.format("checking for default constructor in class '%s'", className));
+ return (T) containerClass.getConstructor().newInstance();
+ } catch (Exception e) {
+ log.debug("no default constructor found");
+ }
+
+ throw new Exception(String.format("no suitable constructor for class '%s'", className));
+ }
+
+ public static <T> T createInstanceFromNamespace(Properties properties, String namespace) throws Exception {
+ return createInstance(getNamespace(properties, namespace));
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <T> T createInstance(Class<?> clazz) throws Exception {
+ try {
+ log.debug(String.format("checking for default constructor in class '%s'", clazz.getSimpleName()));
+ return (T) clazz.getConstructor().newInstance();
+ } catch (Exception e) {
+ log.debug("no default constructor found");
+ }
+
+ throw new Exception(String.format("no suitable constructor for class '%s'", clazz.getSimpleName()));
+ }
+
+ public static <T> T createInstance(String className) throws Exception {
+ return createInstance(Class.forName(className));
+ }
+
+ public static Collection<Properties> getContainerProps(Properties properties) {
+ Collection<Properties> containerProps = Lists.newArrayList();
+
+ String containers = properties.getProperty("containers");
+ String containerTypes[] = StringUtils.split(containers, ",");
+
+ for (String containerType : containerTypes) {
+ Properties containerProp = BootUtils.getNamespace(BootUtils.getNamespace(properties, "container"), containerType);
+ log.debug(String.format("adding container type (type='%s', properties='%s')", containerType, containerProp));
+ containerProps.add(containerProp);
+ }
+
+ return containerProps;
+ }
+
+ private BootUtils() {
+ // left blank
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/Bootstrapper.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/Bootstrapper.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/Bootstrapper.java
new file mode 100644
index 0000000..94de15f
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/Bootstrapper.java
@@ -0,0 +1,93 @@
+package org.apache.helix.metamanager.bootstrap;
+
+import java.util.Properties;
+
+import org.apache.log4j.Logger;
+
+public class Bootstrapper {
+
+ static final Logger log = Logger.getLogger(Bootstrapper.class);
+
+ ManagedCluster managed;
+ MetaCluster meta;
+ ZookeeperWrapper zookeeper;
+ Properties properties;
+
+ public Bootstrapper(Properties properties) {
+ this.properties = properties;
+ }
+
+ public void start() throws Exception {
+ log.info("bootstrapping cluster");
+ if (BootUtils.hasNamespace(properties, "zookeeper")) {
+ log.info("starting zookeeper");
+ zookeeper = new ZookeeperWrapper(BootUtils.getNamespace(properties, "zookeeper"));
+ zookeeper.startService();
+ }
+
+ log.info("starting managed cluster");
+ managed = new ManagedCluster();
+ managed.setProperties(BootUtils.getNamespace(properties, "managed"));
+ managed.start();
+
+ log.info("starting meta cluster");
+ meta = new MetaCluster();
+ meta.setProperties(BootUtils.getNamespace(properties, "meta"));
+ meta.start();
+ }
+
+ public void stop() throws Exception {
+ log.info("tearing down cluster");
+ if (meta != null) {
+ log.info("stopping meta cluster");
+ meta.stop();
+ meta = null;
+ }
+ if (managed != null) {
+ log.info("stopping managed cluster");
+ managed.stop();
+ managed = null;
+ }
+ if (zookeeper != null) {
+ log.info("stopping zookeeper");
+ zookeeper.stopService();
+ zookeeper = null;
+ }
+
+ }
+
+ public ManagedCluster getManaged() {
+ return managed;
+ }
+
+ public MetaCluster getMeta() {
+ return meta;
+ }
+
+ public ZookeeperWrapper getZookeeper() {
+ return zookeeper;
+ }
+
+ public Properties getProperties() {
+ return properties;
+ }
+
+ public static void main(String[] args) throws Exception {
+ String resourcePath = args[0];
+
+ log.info(String.format("reading cluster definition from '%s'", resourcePath));
+ Properties properties = new Properties();
+ properties.load(ClassLoader.getSystemResourceAsStream(resourcePath));
+
+ final Bootstrapper boot = new Bootstrapper(properties);
+ boot.start();
+
+ Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try { boot.stop(); } catch(Exception ignore) {}
+ }
+ }));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/ManagedCluster.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/ManagedCluster.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/ManagedCluster.java
new file mode 100644
index 0000000..b792c9f
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/ManagedCluster.java
@@ -0,0 +1,87 @@
+package org.apache.helix.metamanager.bootstrap;
+
+import java.util.Properties;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.helix.HelixAdmin;
+import org.apache.helix.HelixManager;
+import org.apache.helix.controller.HelixControllerMain;
+import org.apache.helix.manager.zk.ZKHelixAdmin;
+import org.apache.helix.model.IdealState;
+import org.apache.helix.model.IdealState.RebalanceMode;
+import org.apache.helix.model.StateModelDefinition;
+import org.apache.helix.tools.StateModelConfigGenerator;
+import org.apache.log4j.Logger;
+
+public class ManagedCluster {
+
+ static final Logger log = Logger.getLogger(ManagedCluster.class);
+
+ public static final String DEFAULT_CLUSTER = "managed";
+
+ Properties properties;
+
+ HelixAdmin admin;
+ HelixManager controllerMananger;
+
+ public void start() {
+ String cluster = properties.getProperty("cluster", DEFAULT_CLUSTER);
+ String address = properties.getProperty("address");
+
+ log.info(String.format("starting managed cluster service (cluster='%s', address='%s')", cluster, address));
+
+ log.debug("setting up cluster admin");
+ admin = new ZKHelixAdmin(address);
+ admin.addCluster(cluster, false);
+ admin.addStateModelDef(cluster, "OnlineOffline", new StateModelDefinition(StateModelConfigGenerator.generateConfigForOnlineOffline()));
+ admin.addStateModelDef(cluster, "MasterSlave", new StateModelDefinition(StateModelConfigGenerator.generateConfigForMasterSlave()));
+
+ log.debug("setting up resources");
+ String resources = properties.getProperty("resources");
+ String[] resourceNames = StringUtils.split(resources, ",");
+
+ for (String resourceName : resourceNames) {
+ Properties properties = BootUtils.getNamespace(BootUtils.getNamespace(this.properties, "resource"), resourceName);
+
+ log.debug(String.format("parsing resource '%s' (properties='%s')", resourceName, properties));
+
+ String container = properties.getProperty("container");
+ String model = properties.getProperty("model");
+ int partitions = Integer.parseInt(properties.getProperty("partitions"));
+ int replica = Integer.parseInt(properties.getProperty("replica"));
+
+ log.debug(String.format("setting up resource '%s' (container='%s', model='%s', partitions=%d, replica=%d)", resourceName, container, model,
+ partitions, replica));
+
+ admin.addResource(cluster, resourceName, partitions, model, RebalanceMode.FULL_AUTO.toString());
+ IdealState idealState = admin.getResourceIdealState(cluster, resourceName);
+ idealState.setInstanceGroupTag(container);
+ idealState.setReplicas(String.valueOf(replica));
+ admin.setResourceIdealState(cluster, resourceName, idealState);
+ }
+
+ log.debug("setting up controller");
+ controllerMananger = HelixControllerMain.startHelixController(address, cluster, "managedController", HelixControllerMain.STANDALONE);
+ }
+
+ public void stop() {
+ log.info("stopping managed cluster service");
+ if (controllerMananger != null) {
+ controllerMananger.disconnect();
+ controllerMananger = null;
+ }
+ if (admin != null) {
+ admin.close();
+ admin = null;
+ }
+ }
+
+ public Properties getProperties() {
+ return properties;
+ }
+
+ public void setProperties(Properties properties) {
+ this.properties = properties;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/MetaCluster.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/MetaCluster.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/MetaCluster.java
new file mode 100644
index 0000000..51700a8
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/MetaCluster.java
@@ -0,0 +1,201 @@
+package org.apache.helix.metamanager.bootstrap;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.helix.HelixAdmin;
+import org.apache.helix.HelixManager;
+import org.apache.helix.controller.HelixControllerMain;
+import org.apache.helix.manager.zk.ZKHelixAdmin;
+import org.apache.helix.metamanager.ConfigTool;
+import org.apache.helix.metamanager.provider.ProviderProcess;
+import org.apache.helix.metamanager.provider.ProviderRebalancer;
+import org.apache.helix.model.HelixConfigScope;
+import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
+import org.apache.helix.model.IdealState;
+import org.apache.helix.model.IdealState.RebalanceMode;
+import org.apache.helix.model.InstanceConfig;
+import org.apache.helix.model.StateModelDefinition;
+import org.apache.helix.model.builder.HelixConfigScopeBuilder;
+import org.apache.helix.tools.StateModelConfigGenerator;
+import org.apache.log4j.Logger;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+
+public class MetaCluster {
+
+ static final Logger log = Logger.getLogger(MetaCluster.class);
+
+ private static final String DEFAULT_CLUSTER = "meta";
+ private static final String DEFAULT_MANAGED = "managed";
+ private static final String DEFAULT_INTERVAL = "10000";
+
+ Properties properties;
+
+ TargetWrapper target;
+ StatusWrapper status;
+ ProviderWrapper provider;
+
+ HelixAdmin admin;
+ HelixManager controllerManager;
+ ProviderProcess providerProcess;
+
+ String cluster;
+ String address;
+ String managed;
+ int interval;
+
+ ScheduledExecutorService executor;
+
+ public Properties getProperties() {
+ return properties;
+ }
+
+ public void setProperties(Properties properties) {
+ this.properties = properties;
+ }
+
+ public void start() throws Exception {
+ Preconditions.checkArgument(BootUtils.hasNamespace(properties, "target"), "No 'target' property specified");
+ Preconditions.checkArgument(BootUtils.hasNamespace(properties, "status"), "No 'status' property specified");
+ Preconditions.checkArgument(BootUtils.hasNamespace(properties, "provider"), "No 'provider' property specified");
+
+ cluster = properties.getProperty("cluster", DEFAULT_CLUSTER);
+ address = properties.getProperty("address");
+ managed = properties.getProperty("managed", DEFAULT_MANAGED);
+ interval = Integer.valueOf(properties.getProperty("interval", DEFAULT_INTERVAL));
+
+ log.info(String.format("starting meta cluster service (cluster='%s', address='%s', managed='%s')", cluster, address, managed));
+
+ log.debug("setting up cluster admin");
+ admin = new ZKHelixAdmin(address);
+ admin.addCluster(cluster, false);
+ admin.addStateModelDef(cluster, "OnlineOffline", new StateModelDefinition(StateModelConfigGenerator.generateConfigForOnlineOffline()));
+
+ log.debug("setting up target service");
+ target = new TargetWrapper(BootUtils.getNamespace(properties, "target"));
+ target.startService();
+
+ log.debug("setting up container status service");
+ status = new StatusWrapper(BootUtils.getNamespace(properties, "status"));
+ status.startService();
+
+ log.debug("setting up container provider service");
+ provider = new ProviderWrapper(BootUtils.getNamespace(properties, "provider"));
+ admin.addInstance(cluster, new InstanceConfig(provider.getProviderName()));
+
+ provider.startService();
+
+ log.debug("setting up config tool");
+ ConfigTool.setTargetProvider(target.getTarget());
+ ConfigTool.setStatusProvider(status.getStatus());
+
+ log.debug("setting up provider");
+ String providerName = provider.getProviderName();
+
+ admin.addInstance(cluster, new InstanceConfig(providerName));
+
+ for (String containerType : provider.getContainerTypes()) {
+ log.debug(String.format("setting up container type '%s'", containerType));
+
+ admin.addResource(cluster, containerType, target.getTarget().getTargetContainerCount(containerType), "OnlineOffline",
+ RebalanceMode.USER_DEFINED.toString());
+
+ IdealState idealState = admin.getResourceIdealState(cluster, containerType);
+ idealState.setRebalancerClassName(ProviderRebalancer.class.getName());
+ idealState.setReplicas("1");
+
+ // BEGIN workaround
+ // FIXME workaround for HELIX-226
+ Map<String, List<String>> listFields = Maps.newHashMap();
+ Map<String, Map<String, String>> mapFields = Maps.newHashMap();
+ for(int i=0; i<256; i++) {
+ String partitionName = containerType + "_" + i;
+ listFields.put(partitionName, new ArrayList<String>());
+ mapFields.put(partitionName, new HashMap<String, String>());
+ }
+ idealState.getRecord().setListFields(listFields);
+ idealState.getRecord().setMapFields(mapFields);
+ // END workaround
+
+ admin.setResourceIdealState(cluster, containerType, idealState);
+ }
+
+ log.debug("starting controller");
+ controllerManager = HelixControllerMain.startHelixController(address, cluster, "metaController", HelixControllerMain.STANDALONE);
+
+ log.debug("starting state refresh service");
+ executor = Executors.newSingleThreadScheduledExecutor();
+ executor.scheduleAtFixedRate(new MetaRefreshRunnable(), interval, interval, TimeUnit.MILLISECONDS);
+
+ HelixConfigScope scope = new HelixConfigScopeBuilder(ConfigScopeProperty.CLUSTER, cluster).build();
+ admin.setConfig(scope, Collections.singletonMap("key", "value"));
+
+ }
+
+ public void stop() throws Exception {
+ log.info("stopping meta cluster service");
+ if (executor != null) {
+ executor.shutdownNow();
+ executor = null;
+ }
+ if (controllerManager != null) {
+ controllerManager.disconnect();
+ controllerManager = null;
+ }
+ if (providerProcess != null) {
+ providerProcess.stop();
+ providerProcess = null;
+ }
+ if (provider != null) {
+ provider.stopService();
+ provider = null;
+ }
+ if (status != null) {
+ status.stopService();
+ status = null;
+ }
+ if (target != null) {
+ target.stopService();
+ target = null;
+ }
+ if (admin != null) {
+ admin.close();
+ admin = null;
+ }
+ }
+
+ public TargetWrapper getTarget() {
+ return target;
+ }
+
+ public StatusWrapper getStatus() {
+ return status;
+ }
+
+ public ProviderWrapper getProvider() {
+ return provider;
+ }
+
+ private class MetaRefreshRunnable implements Runnable {
+ @Override
+ public void run() {
+ log.debug("running status refresh");
+ for (String containerType : provider.getContainerTypes()) {
+ log.debug(String.format("refreshing container type '%s'", containerType));
+
+ IdealState poke = admin.getResourceIdealState(cluster, containerType);
+ admin.setResourceIdealState(cluster, containerType, poke);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/ProviderWrapper.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/ProviderWrapper.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/ProviderWrapper.java
new file mode 100644
index 0000000..b8e35bb
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/ProviderWrapper.java
@@ -0,0 +1,162 @@
+package org.apache.helix.metamanager.bootstrap;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Properties;
+import java.util.Set;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.helix.metamanager.impl.local.LocalContainerProviderProcess;
+import org.apache.helix.metamanager.impl.shell.ShellContainerProviderProcess;
+import org.apache.helix.metamanager.impl.yarn.YarnContainerProviderProcess;
+import org.apache.helix.metamanager.impl.yarn.YarnContainerProviderProperties;
+import org.apache.log4j.Logger;
+
+public class ProviderWrapper {
+
+ static final Logger log = Logger.getLogger(ProviderWrapper.class);
+
+ WrapperImpl impl;
+ Properties properties;
+
+ public ProviderWrapper(Properties properties) {
+ this.properties = properties;
+ }
+
+ public void startService() throws Exception {
+ String type = (String) properties.get("type");
+
+ log.info(String.format("starting container provider service (type='%s')", type));
+
+ if ("local".equals(type)) {
+ impl = new LocalWrapperImpl();
+
+ } else if ("shell".equals(type)) {
+ impl = new ShellWrapperImpl();
+
+ } else if ("yarn".equals(type)) {
+ impl = new YarnWrapperImpl();
+
+ } else {
+ throw new IllegalArgumentException(String.format("type '%s' not supported", type));
+ }
+
+ impl.startService();
+ }
+
+ public void stopService() throws Exception {
+ impl.stopService();
+ }
+
+ public String getProviderName() {
+ return properties.getProperty("name");
+ }
+
+ public Set<String> getContainerTypes() {
+ String containers = properties.getProperty("containers");
+ String containerTypes[] = StringUtils.split(containers, ",");
+ return new HashSet<String>(Arrays.asList(containerTypes));
+ }
+
+ static interface WrapperImpl {
+ void startService() throws Exception;
+
+ void stopService() throws Exception;
+ }
+
+ class LocalWrapperImpl implements WrapperImpl {
+ LocalContainerProviderProcess process;
+
+ @Override
+ public void startService() throws Exception {
+ String name = properties.getProperty("name");
+ String address = properties.getProperty("address");
+ String cluster = properties.getProperty("cluster");
+ String containers = properties.getProperty("containers");
+
+ log.debug(String.format("creating local container provider (name='%s', address='%s', cluster='%s', containers='%s')", name, address, cluster,
+ containers));
+
+ process = new LocalContainerProviderProcess();
+ process.configure(properties);
+ process.start();
+ }
+
+ @Override
+ public void stopService() throws Exception {
+ process.stop();
+ process = null;
+ }
+
+ }
+
+ class ShellWrapperImpl implements WrapperImpl {
+
+ ShellContainerProviderProcess process;
+
+ @Override
+ public void startService() throws Exception {
+ String name = properties.getProperty("name");
+ String address = properties.getProperty("address");
+ String cluster = properties.getProperty("cluster");
+ String containers = properties.getProperty("containers");
+
+ log.debug(String.format("creating shell container provider (name='%s', address='%s', cluster='%s', containers='%s')", name, address, cluster,
+ containers));
+
+ process = new ShellContainerProviderProcess();
+ process.configure(properties);
+ process.start();
+ }
+
+ @Override
+ public void stopService() throws Exception {
+ process.stop();
+ process = null;
+ }
+
+ }
+
+ class YarnWrapperImpl implements WrapperImpl {
+
+ YarnContainerProviderProcess process;
+
+ @Override
+ public void startService() throws Exception {
+ String name = properties.getProperty("name");
+ String address = properties.getProperty("address");
+ String cluster = properties.getProperty("cluster");
+ String containers = properties.getProperty("containers");
+ String metadata = properties.getProperty("metadata");
+ String resourcemanager = properties.getProperty("resourcemananger");
+ String scheduler = properties.getProperty("scheduler");
+ String user = properties.getProperty("user");
+ String hdfs = properties.getProperty("hdfs");
+
+ YarnContainerProviderProperties yarnProperties = new YarnContainerProviderProperties();
+ yarnProperties.setProperty(YarnContainerProviderProperties.CLUSTER, cluster);
+ yarnProperties.setProperty(YarnContainerProviderProperties.ADDRESS, address);
+ yarnProperties.setProperty(YarnContainerProviderProperties.NAME, name);
+ yarnProperties.setProperty(YarnContainerProviderProperties.METADATA, metadata);
+ yarnProperties.setProperty(YarnContainerProviderProperties.RESOURCEMANAGER, resourcemanager);
+ yarnProperties.setProperty(YarnContainerProviderProperties.SCHEDULER, scheduler);
+ yarnProperties.setProperty(YarnContainerProviderProperties.USER, user);
+ yarnProperties.setProperty(YarnContainerProviderProperties.HDFS, hdfs);
+
+ log.debug(String.format("creating yarn container provider (name='%s', address='%s', cluster='%s', metadata='%s', resourcemananger='%s', " +
+ "scheduler='%s', user='%s', hdfs='%s', containers='%s')", name, address, cluster, metadata, resourcemanager, scheduler, user, hdfs, containers));
+
+ process = new YarnContainerProviderProcess();
+ process.configure(yarnProperties);
+ process.start();
+ }
+
+ @Override
+ public void stopService() throws Exception {
+ process.stop();
+ process = null;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/StatusWrapper.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/StatusWrapper.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/StatusWrapper.java
new file mode 100644
index 0000000..20fa0db
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/StatusWrapper.java
@@ -0,0 +1,122 @@
+package org.apache.helix.metamanager.bootstrap;
+
+import java.util.Properties;
+
+import org.apache.helix.metamanager.StatusProvider;
+import org.apache.helix.metamanager.StatusProviderService;
+import org.apache.helix.metamanager.impl.local.LocalStatusProvider;
+import org.apache.helix.metamanager.impl.shell.ShellStatusProvider;
+import org.apache.helix.metamanager.impl.yarn.YarnStatusProvider;
+import org.apache.log4j.Logger;
+
+public class StatusWrapper {
+
+ static final Logger log = Logger.getLogger(StatusWrapper.class);
+
+ WrapperImpl impl;
+ StatusProviderService status;
+ Properties properties;
+
+ public StatusWrapper(Properties properties) {
+ this.properties = properties;
+ }
+
+ public void startService() throws Exception {
+ String type = (String) properties.get("type");
+
+ log.info(String.format("starting container status service (type='%s')", type));
+
+ if ("local".equals(type)) {
+ impl = new LocalWrapperImpl();
+
+ } else if ("shell".equals(type)) {
+ impl = new ShellWrapperImpl();
+
+ } else if ("yarn".equals(type)) {
+ impl = new YarnWrapperImpl();
+
+ } else {
+ throw new IllegalArgumentException(String.format("type '%s' not supported", type));
+ }
+
+ impl.startService();
+ }
+
+ public void stopService() throws Exception {
+ log.debug("stopping container status provider");
+ impl.stopService();
+ status = null;
+ }
+
+ public StatusProvider getStatus() {
+ return status;
+ }
+
+ static interface WrapperImpl {
+ void startService() throws Exception;
+
+ void stopService() throws Exception;
+ }
+
+ class LocalWrapperImpl implements WrapperImpl {
+
+ LocalStatusProvider status;
+
+ @Override
+ public void startService() throws Exception {
+ log.debug("creating local container status provider");
+ status = new LocalStatusProvider();
+ status.configure(properties);
+ status.start();
+
+ StatusWrapper.this.status = status;
+ }
+
+ @Override
+ public void stopService() throws Exception {
+ status.stop();
+ }
+ }
+
+ class ShellWrapperImpl implements WrapperImpl {
+
+ ShellStatusProvider status;
+
+ @Override
+ public void startService() throws Exception {
+ log.debug("creating shell container status provider");
+ status = new ShellStatusProvider();
+ status.configure(properties);
+ status.start();
+ StatusWrapper.this.status = status;
+ }
+
+ @Override
+ public void stopService() throws Exception {
+ status.stop();
+ }
+ }
+
+ class YarnWrapperImpl implements WrapperImpl {
+
+ YarnStatusProvider status;
+
+ @Override
+ public void startService() throws Exception {
+ String metadata = properties.getProperty("metadata");
+
+ log.debug(String.format("creating yarn container status provider (metadata='%s')", metadata));
+ status = new YarnStatusProvider();
+ status.configure(properties);
+ status.start();
+
+ StatusWrapper.this.status = status;
+ }
+
+ @Override
+ public void stopService() throws Exception {
+ status.stop();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/TargetWrapper.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/TargetWrapper.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/TargetWrapper.java
new file mode 100644
index 0000000..5920fc4
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/TargetWrapper.java
@@ -0,0 +1,117 @@
+package org.apache.helix.metamanager.bootstrap;
+
+import java.util.Properties;
+
+import org.apache.helix.metamanager.TargetProvider;
+import org.apache.helix.metamanager.TargetProviderService;
+import org.apache.helix.metamanager.impl.FileTargetProvider;
+import org.apache.helix.metamanager.impl.RedisTargetProvider;
+import org.apache.helix.metamanager.impl.StaticTargetProvider;
+import org.apache.log4j.Logger;
+
+public class TargetWrapper {
+
+ static final Logger log = Logger.getLogger(TargetWrapper.class);
+
+ WrapperImpl impl;
+ Properties properties;
+ TargetProviderService target;
+
+ public TargetWrapper(Properties properties) {
+ this.properties = properties;
+ }
+
+ public void startService() throws Exception {
+ String type = (String) properties.get("type");
+
+ log.info(String.format("starting target service (type='%s')", type));
+
+ if ("static".equals(type)) {
+ impl = new StaticWrapperImpl();
+
+ } else if ("file".equals(type)) {
+ impl = new FileWrapperImpl();
+
+ } else if ("redis".equals(type)) {
+ impl = new RedisWrapperImpl();
+
+ } else {
+ throw new IllegalArgumentException(String.format("type '%s' not supported", type));
+ }
+
+ impl.startService();
+ }
+
+ public void stopService() throws Exception {
+ log.info("stopping target service");
+ impl.stopService();
+ target = null;
+ }
+
+ public TargetProvider getTarget() {
+ return target;
+ }
+
+ static interface WrapperImpl {
+ void startService() throws Exception;
+
+ void stopService() throws Exception;
+ }
+
+ private class StaticWrapperImpl implements WrapperImpl {
+ @Override
+ public void startService() throws Exception {
+ log.debug("creating static target provider");
+ Properties prop = new Properties();
+ prop.putAll(properties);
+ prop.remove("type");
+
+ target = new StaticTargetProvider();
+ target.configure(prop);
+ target.start();
+ }
+
+ @Override
+ public void stopService() throws Exception {
+ target.stop();
+ }
+ }
+
+ private class FileWrapperImpl implements WrapperImpl {
+ @Override
+ public void startService() throws Exception {
+ log.debug("creating file target provider");
+ Properties prop = new Properties();
+ prop.putAll(properties);
+ prop.remove("type");
+
+ target = new FileTargetProvider();
+ target.configure(prop);
+ target.start();
+ }
+
+ @Override
+ public void stopService() throws Exception {
+ target.stop();
+ }
+ }
+
+ private class RedisWrapperImpl implements WrapperImpl {
+ @Override
+ public void startService() throws Exception {
+ log.debug("creating redis target provider");
+ Properties prop = new Properties();
+ prop.putAll(properties);
+ prop.remove("type");
+
+ target = new RedisTargetProvider();
+ target.configure(prop);
+ target.start();
+ }
+
+ @Override
+ public void stopService() throws Exception {
+ target.stop();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/ZookeeperWrapper.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/ZookeeperWrapper.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/ZookeeperWrapper.java
new file mode 100644
index 0000000..eca7fab
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrap/ZookeeperWrapper.java
@@ -0,0 +1,57 @@
+package org.apache.helix.metamanager.bootstrap;
+
+import java.io.File;
+import java.util.Properties;
+
+import org.I0Itec.zkclient.IDefaultNameSpace;
+import org.I0Itec.zkclient.ZkClient;
+import org.I0Itec.zkclient.ZkServer;
+import org.apache.commons.io.FileUtils;
+import org.apache.log4j.Logger;
+
+public class ZookeeperWrapper {
+
+ static final Logger log = Logger.getLogger(ZookeeperWrapper.class);
+
+ ZkServer server;
+ Properties properties;
+
+ public ZookeeperWrapper(Properties properties) {
+ this.properties = properties;
+ }
+
+ public void startService() {
+ String dataDir = properties.getProperty("datadir");
+ String logDir = properties.getProperty("logdir");
+ int port = Integer.parseInt(properties.getProperty("port"));
+
+ log.info(String.format("starting zookeeper service (dataDir='%s', logDir='%s', port=%d)", dataDir, logDir, port));
+
+ FileUtils.deleteQuietly(new File(dataDir));
+ FileUtils.deleteQuietly(new File(logDir));
+
+ IDefaultNameSpace defaultNameSpace = new IDefaultNameSpace() {
+ @Override
+ public void createDefaultNameSpace(ZkClient zkClient) {
+ // left blank
+ }
+ };
+
+ server = new ZkServer(dataDir, logDir, defaultNameSpace, port);
+ server.start();
+ }
+
+ public void stopService() {
+ log.info("stopping zookeeper service");
+
+ if (server != null) {
+ server.shutdown();
+ server = null;
+ }
+ }
+
+ public ZkServer getZookeeper() {
+ return server;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/Boot.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/Boot.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/Boot.java
new file mode 100644
index 0000000..004573d
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/Boot.java
@@ -0,0 +1,132 @@
+package org.apache.helix.metamanager.bootstrapper;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.helix.metamanager.Service;
+import org.apache.log4j.Logger;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+/**
+ * Bootstrapper for elastic cluster deployment using *.properties configuration
+ * files. (Program entry point)
+ *
+ */
+public class Boot implements Service {
+
+ static final Logger log = Logger.getLogger(Boot.class);
+
+ static final Map<String, Class<? extends Service>> classes = new HashMap<String, Class<? extends Service>>();
+ static {
+ classes.put("zookeeper", ZookeeperService.class);
+ classes.put("cluster", ClusterService.class);
+ classes.put("resource", ResourceService.class);
+ classes.put("controller", ControllerService.class);
+ classes.put("metacluster", MetaClusterService.class);
+ classes.put("metaresource", MetaResourceService.class);
+ classes.put("metaprovider", MetaProviderService.class);
+ classes.put("metacontroller", MetaControllerService.class);
+ }
+
+ static final List<String> serviceOrder = Arrays.asList("zookeeper", "cluster", "resource", "metacluster", "metaresource",
+ "metaprovider", "controller", "metacontroller");
+
+ Properties properties;
+ List<Service> services = Lists.newArrayList();
+
+ @Override
+ public void configure(Properties properties) throws Exception {
+ Preconditions.checkNotNull(properties);
+ this.properties = properties;
+ }
+
+ @Override
+ public void start() throws Exception {
+ log.info(String.format("bootstraping started"));
+
+ for (String key : serviceOrder) {
+ if (BootUtils.hasNamespace(properties, key + ".0")) {
+ processIndexedNamespace(key);
+ } else if (BootUtils.hasNamespace(properties, key)) {
+ processNamespace(key);
+ }
+ }
+
+ log.info(String.format("bootstraping completed"));
+ }
+
+ private void processIndexedNamespace(String key) throws Exception {
+ int i = 0;
+ String indexedKey = key + "." + i;
+
+ while (BootUtils.hasNamespace(properties, indexedKey)) {
+ log.info(String.format("processing namespace '%s'", indexedKey));
+ Service service = BootUtils.createInstance(classes.get(key));
+ service.configure(BootUtils.getNamespace(properties, indexedKey));
+ service.start();
+
+ services.add(service);
+
+ i++;
+ indexedKey = key + "." + i;
+ }
+ }
+
+ private void processNamespace(String key) throws Exception {
+ log.info(String.format("processing namespace '%s'", key));
+ Service service = BootUtils.createInstance(classes.get(key));
+ service.configure(BootUtils.getNamespace(properties, key));
+ service.start();
+
+ services.add(service);
+ }
+
+ @Override
+ public void stop() throws Exception {
+ log.info(String.format("shutdown started"));
+
+ Collections.reverse(services);
+ for (Service service : services) {
+ service.stop();
+ }
+
+ log.info(String.format("shutdown completed"));
+ }
+
+ public Collection<Service> getServcies() {
+ return services;
+ }
+
+ public static void main(String[] args) throws Exception {
+ if (args.length < 1) {
+ log.error(String.format("Usage: Boot properties_path"));
+ return;
+ }
+
+ String resourcePath = args[0];
+
+ log.info(String.format("reading definition from '%s'", resourcePath));
+ Properties properties = new Properties();
+ properties.load(ClassLoader.getSystemResourceAsStream(resourcePath));
+
+ final Boot boot = new Boot();
+ boot.configure(properties);
+ boot.start();
+
+ Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+ @Override
+ public void run() {
+ log.debug("Running shutdown hook");
+ try { boot.stop(); } catch (Exception ignore) {}
+ }
+ }));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/BootUtils.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/BootUtils.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/BootUtils.java
new file mode 100644
index 0000000..2fb9ff6
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/BootUtils.java
@@ -0,0 +1,104 @@
+package org.apache.helix.metamanager.bootstrapper;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+/**
+ * Utility for instantiating bootstrapping services and parsing hierarchical
+ * properties files.
+ *
+ */
+public class BootUtils {
+
+ public static final String CLASS_PROPERTY = "class";
+ static final Logger log = Logger.getLogger(BootUtils.class);
+
+ public static boolean hasNamespace(Properties properties, String namespace) {
+ String prefix = namespace + ".";
+ for (String key : properties.stringPropertyNames()) {
+ if (key.startsWith(prefix))
+ return true;
+ }
+ return false;
+ }
+
+ public static Set<String> getNamespaces(Properties properties) {
+ Pattern pattern = Pattern.compile("^([^\\.\\=]+)");
+
+ Set<String> namespaces = Sets.newHashSet();
+
+ for (Map.Entry<Object, Object> rawEntry : properties.entrySet()) {
+ String key = (String) rawEntry.getKey();
+
+ Matcher matcher = pattern.matcher(key);
+ if (matcher.find()) {
+ namespaces.add(matcher.group(1));
+ }
+ }
+
+ return namespaces;
+ }
+
+ public static Properties getNamespace(Properties source, String namespace) {
+ Properties dest = new Properties();
+ String prefix = namespace + ".";
+
+ for (Map.Entry<Object, Object> rawEntry : source.entrySet()) {
+ String key = (String) rawEntry.getKey();
+ String value = (String) rawEntry.getValue();
+
+ if (key.startsWith(prefix)) {
+ String newKey = key.substring(prefix.length());
+ dest.put(newKey, value);
+ }
+ }
+
+ return dest;
+ }
+
+ public static Collection<Properties> getContainerProps(Properties properties) {
+ Collection<Properties> containerProps = Lists.newArrayList();
+
+ String containers = properties.getProperty("containers");
+ String containerTypes[] = StringUtils.split(containers, ",");
+
+ for (String containerType : containerTypes) {
+ Properties containerProp = BootUtils.getNamespace(BootUtils.getNamespace(properties, "container"), containerType);
+ log.debug(String.format("adding container type (type='%s', properties='%s')", containerType, containerProp));
+ containerProps.add(containerProp);
+ }
+
+ return containerProps;
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <T> T createInstance(Class<?> clazz) throws Exception {
+ try {
+ log.debug(String.format("checking for default constructor in class '%s'", clazz.getSimpleName()));
+ return (T) clazz.getConstructor().newInstance();
+ } catch (Exception e) {
+ log.debug("no default constructor found");
+ }
+
+ throw new Exception(String.format("no suitable constructor for class '%s'", clazz.getSimpleName()));
+ }
+
+ public static <T> T createInstance(String className) throws Exception {
+ return createInstance(Class.forName(className));
+ }
+
+ private BootUtils() {
+ // left blank
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/ClusterService.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/ClusterService.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/ClusterService.java
new file mode 100644
index 0000000..5b3ec7e
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/ClusterService.java
@@ -0,0 +1,46 @@
+package org.apache.helix.metamanager.bootstrapper;
+
+import java.util.Properties;
+
+import org.apache.helix.HelixAdmin;
+import org.apache.helix.manager.zk.ZKHelixAdmin;
+import org.apache.helix.metamanager.Service;
+import org.apache.helix.model.StateModelDefinition;
+import org.apache.helix.tools.StateModelConfigGenerator;
+import org.apache.log4j.Logger;
+
+/**
+ * Cluster bootstrapping. Create Helix data structures in zookeeper for the
+ * managed cluster.
+ *
+ */
+public class ClusterService implements Service {
+
+ static final Logger log = Logger.getLogger(ClusterService.class);
+
+ String name;
+ String address;
+
+ @Override
+ public void configure(Properties properties) throws Exception {
+ name = properties.getProperty("name", "cluster");
+ address = properties.getProperty("address", "localhost:2199");
+ }
+
+ @Override
+ public void start() throws Exception {
+ log.info(String.format("setting up '%s/%s'", address, name));
+ HelixAdmin admin = new ZKHelixAdmin(address);
+ admin.addCluster(name, false);
+ admin.addStateModelDef(name, "OnlineOffline", new StateModelDefinition(StateModelConfigGenerator.generateConfigForOnlineOffline()));
+ admin.addStateModelDef(name, "MasterSlave", new StateModelDefinition(StateModelConfigGenerator.generateConfigForMasterSlave()));
+ admin.close();
+ log.info("setup complete");
+ }
+
+ @Override
+ public void stop() throws Exception {
+ // left blank
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/ControllerService.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/ControllerService.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/ControllerService.java
new file mode 100644
index 0000000..2a95ecf
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/ControllerService.java
@@ -0,0 +1,50 @@
+package org.apache.helix.metamanager.bootstrapper;
+
+import java.util.Properties;
+import java.util.concurrent.ScheduledExecutorService;
+
+import org.apache.helix.HelixManager;
+import org.apache.helix.controller.HelixControllerMain;
+import org.apache.helix.metamanager.Service;
+import org.apache.log4j.Logger;
+
+/**
+ * Helix controller bootstrapping and management. Create standalone controller
+ * for managed Helix cluster.
+ *
+ */
+public class ControllerService implements Service {
+
+ static final Logger log = Logger.getLogger(ControllerService.class);
+
+ String name;
+ String cluster;
+ String address;
+
+ HelixManager manager;
+
+ ScheduledExecutorService executor;
+
+ @Override
+ public void configure(Properties properties) throws Exception {
+ name = properties.getProperty("name", "controller");
+ cluster = properties.getProperty("cluster", "cluster");
+ address = properties.getProperty("address", "localhost:2199");
+ }
+
+ @Override
+ public void start() throws Exception {
+ log.info(String.format("starting controller '%s' at '%s/%s'", name, address, cluster));
+ manager = HelixControllerMain.startHelixController(address, cluster, name, HelixControllerMain.STANDALONE);
+ }
+
+ @Override
+ public void stop() throws Exception {
+ if (manager != null) {
+ log.info(String.format("stopping controller '%s' at '%s/%s'", name, address, cluster));
+ manager.disconnect();
+ manager = null;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/MetaClusterService.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/MetaClusterService.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/MetaClusterService.java
new file mode 100644
index 0000000..340c961
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/MetaClusterService.java
@@ -0,0 +1,61 @@
+package org.apache.helix.metamanager.bootstrapper;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.helix.HelixAdmin;
+import org.apache.helix.manager.zk.ZKHelixAdmin;
+import org.apache.helix.metamanager.Service;
+import org.apache.helix.model.HelixConfigScope;
+import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
+import org.apache.helix.model.StateModelDefinition;
+import org.apache.helix.model.builder.HelixConfigScopeBuilder;
+import org.apache.helix.tools.StateModelConfigGenerator;
+import org.apache.log4j.Logger;
+
+/**
+ * Meta cluster bootstrapping. Create Helix data structures in zookeeper for
+ * the meta cluster.
+ *
+ */
+public class MetaClusterService implements Service {
+
+ static final Logger log = Logger.getLogger(MetaClusterService.class);
+
+ String name;
+ String address;
+ String managedCluster;
+ String managedAddress;
+
+ @Override
+ public void configure(Properties properties) throws Exception {
+ name = properties.getProperty("name", "metacluster");
+ address = properties.getProperty("address", "localhost:2199");
+ managedCluster = properties.getProperty("managedcluster", "cluster");
+ managedAddress = properties.getProperty("managedaddress", "localhost:2199");
+ }
+
+ @Override
+ public void start() throws Exception {
+ log.info(String.format("setting up '%s/%s'", address, name));
+ HelixAdmin admin = new ZKHelixAdmin(address);
+ admin.addCluster(name, false);
+ admin.addStateModelDef(name, "OnlineOffline", new StateModelDefinition(StateModelConfigGenerator.generateConfigForOnlineOffline()));
+
+ HelixConfigScope scope = new HelixConfigScopeBuilder(ConfigScopeProperty.CLUSTER, name).build();
+ Map<String, String> properties = new HashMap<String, String>();
+ properties.put("cluster", managedCluster);
+ properties.put("address", managedAddress);
+ admin.setConfig(scope, properties);
+
+ admin.close();
+ log.info("setup complete");
+ }
+
+ @Override
+ public void stop() throws Exception {
+ // left blank
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/MetaControllerService.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/MetaControllerService.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/MetaControllerService.java
new file mode 100644
index 0000000..a12753c
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/MetaControllerService.java
@@ -0,0 +1,114 @@
+package org.apache.helix.metamanager.bootstrapper;
+
+import java.util.Properties;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.helix.HelixAdmin;
+import org.apache.helix.HelixManager;
+import org.apache.helix.controller.HelixControllerMain;
+import org.apache.helix.metamanager.Service;
+import org.apache.helix.metamanager.StatusProviderService;
+import org.apache.helix.metamanager.TargetProviderService;
+import org.apache.helix.metamanager.provider.ProviderRebalancerSingleton;
+import org.apache.helix.model.IdealState;
+import org.apache.log4j.Logger;
+
+/**
+ * Meta cluster controller bootstrapping and management. Create standalone
+ * controller for Helix meta cluster. Spawn StatusProvider and TargetProvider
+ * and trigger periodic status refresh in meta cluster.
+ *
+ */
+public class MetaControllerService implements Service {
+
+ static final Logger log = Logger.getLogger(MetaControllerService.class);
+
+ String name;
+ String metacluster;
+ String metaaddress;
+ long autorefresh;
+
+ HelixManager manager;
+ StatusProviderService statusService;
+ TargetProviderService targetService;
+ ScheduledExecutorService executor;
+
+ @Override
+ public void configure(Properties properties) throws Exception {
+ name = properties.getProperty("name", "controller");
+ metacluster = properties.getProperty("metacluster", "metacluster");
+ metaaddress = properties.getProperty("metaaddress", "localhost:2199");
+ autorefresh = Long.valueOf(properties.getProperty("autorefresh", "0"));
+
+ Properties statusProperties = BootUtils.getNamespace(properties, "status");
+ statusService = BootUtils.createInstance(Class.forName(statusProperties.getProperty("class")));
+ statusService.configure(statusProperties);
+ ProviderRebalancerSingleton.setStatusProvider(statusService);
+
+ Properties targetProperties = BootUtils.getNamespace(properties, "target");
+ targetService = BootUtils.createInstance(Class.forName(targetProperties.getProperty("class")));
+ targetService.configure(targetProperties);
+ ProviderRebalancerSingleton.setTargetProvider(targetService);
+ }
+
+ @Override
+ public void start() throws Exception {
+ log.debug("Starting status service");
+ statusService.start();
+
+ log.debug("Starting target service");
+ targetService.start();
+
+ log.info(String.format("starting controller '%s' at '%s/%s'", name, metaaddress, metacluster));
+ manager = HelixControllerMain.startHelixController(metaaddress, metacluster, name, HelixControllerMain.STANDALONE);
+
+ if (autorefresh > 0) {
+ log.debug(String.format("installing autorefresh with interval %d ms", autorefresh));
+ executor = Executors.newSingleThreadScheduledExecutor();
+ executor.scheduleAtFixedRate(new RefreshRunnable(), autorefresh, autorefresh, TimeUnit.MILLISECONDS);
+ }
+ }
+
+ @Override
+ public void stop() throws Exception {
+ if (executor != null) {
+ executor.shutdownNow();
+ while (!executor.isTerminated()) {
+ Thread.sleep(100);
+ }
+ executor = null;
+ }
+ if (manager != null) {
+ log.info(String.format("Stopping controller '%s' at '%s/%s'", name, metaaddress, metacluster));
+ manager.disconnect();
+ manager = null;
+ }
+ if (targetService != null) {
+ log.debug("Stopping target service");
+ targetService.stop();
+ targetService = null;
+ }
+ if (statusService != null) {
+ log.debug("Stopping status service");
+ statusService.stop();
+ statusService = null;
+ }
+ }
+
+ private class RefreshRunnable implements Runnable {
+ @Override
+ public void run() {
+ log.debug("running status refresh");
+ HelixAdmin admin = manager.getClusterManagmentTool();
+
+ for (String metaResource : admin.getResourcesInCluster(metacluster)) {
+ log.debug(String.format("refreshing meta resource '%s'", metaResource));
+
+ IdealState poke = admin.getResourceIdealState(metacluster, metaResource);
+ admin.setResourceIdealState(metacluster, metaResource, poke);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/MetaProviderService.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/MetaProviderService.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/MetaProviderService.java
new file mode 100644
index 0000000..0b68580
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/MetaProviderService.java
@@ -0,0 +1,81 @@
+package org.apache.helix.metamanager.bootstrapper;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.helix.HelixAdmin;
+import org.apache.helix.manager.zk.ZKHelixAdmin;
+import org.apache.helix.metamanager.Service;
+import org.apache.helix.metamanager.provider.ProviderProperties;
+import org.apache.helix.model.HelixConfigScope;
+import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
+import org.apache.helix.model.builder.HelixConfigScopeBuilder;
+import org.apache.log4j.Logger;
+
+import com.google.common.collect.Lists;
+
+/**
+ * ContainerProvider bootstrapping and management. Create container provider
+ * participant, configure with container properties from meta resources and
+ * connect to meta cluster.
+ *
+ */
+public class MetaProviderService implements Service {
+
+ static final Logger log = Logger.getLogger(MetaProviderService.class);
+
+ Service service;
+
+ String clazz;
+ String metaAddress;
+ String metaCluster;
+
+ ProviderProperties config;
+
+ @Override
+ public void configure(Properties properties) throws Exception {
+ clazz = properties.getProperty("class");
+ metaAddress = properties.getProperty("metaaddress", "localhost:2199");
+ metaCluster = properties.getProperty("metacluster", "metacluster");
+
+ config = new ProviderProperties();
+ config.putAll(properties);
+ }
+
+ @Override
+ public void start() throws Exception {
+ log.info(String.format("starting service '%s' (config=%s)", clazz, config));
+
+ HelixAdmin admin = new ZKHelixAdmin(metaAddress);
+
+ HelixConfigScope managedScope = new HelixConfigScopeBuilder(ConfigScopeProperty.CLUSTER, metaCluster).build();
+ Map<String, String> managedProps = admin.getConfig(managedScope, Lists.newArrayList("cluster", "address"));
+ config.putAll(managedProps);
+
+ for (String resource : admin.getResourcesInCluster(metaCluster)) {
+ HelixConfigScope resScope = new HelixConfigScopeBuilder(ConfigScopeProperty.RESOURCE, metaCluster, resource).build();
+ List<String> resKeys = admin.getConfigKeys(resScope);
+ Map<String, String> resProps = admin.getConfig(resScope, resKeys);
+
+ Properties properties = new Properties();
+ properties.putAll(resProps);
+
+ config.addContainer(resource, properties);
+ }
+
+ service = BootUtils.createInstance(clazz);
+ service.configure(config);
+ service.start();
+ }
+
+ @Override
+ public void stop() throws Exception {
+ log.info(String.format("stopping service '%s' (config=%s)", clazz, config));
+ if (service != null) {
+ service.stop();
+ service = null;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/MetaResourceService.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/MetaResourceService.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/MetaResourceService.java
new file mode 100644
index 0000000..c8f0664
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/MetaResourceService.java
@@ -0,0 +1,87 @@
+package org.apache.helix.metamanager.bootstrapper;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.helix.HelixAdmin;
+import org.apache.helix.manager.zk.ZKHelixAdmin;
+import org.apache.helix.metamanager.Service;
+import org.apache.helix.metamanager.provider.ProviderRebalancer;
+import org.apache.helix.model.HelixConfigScope;
+import org.apache.helix.model.IdealState;
+import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
+import org.apache.helix.model.IdealState.RebalanceMode;
+import org.apache.helix.model.builder.HelixConfigScopeBuilder;
+import org.apache.log4j.Logger;
+
+import com.google.common.collect.Maps;
+
+/**
+ * Bootstrapping meta resource. Create container type configuration in Helix
+ * zookeeper namespace.
+ *
+ */
+public class MetaResourceService implements Service {
+
+ static final Logger log = Logger.getLogger(MetaResourceService.class);
+
+ String metaCluster;
+ String metaAddress;
+ String name;
+ Map<String, String> config;
+
+ @Override
+ public void configure(Properties properties) throws Exception {
+ metaCluster = properties.getProperty("metacluster", "metacluster");
+ metaAddress = properties.getProperty("metaaddress", "localhost:2199");
+ name = properties.getProperty("name", "container");
+
+ this.config = new HashMap<String, String>();
+ for (Map.Entry<Object, Object> entry : properties.entrySet()) {
+ this.config.put((String) entry.getKey(), (String) entry.getValue());
+ }
+ }
+
+ @Override
+ public void start() throws Exception {
+ log.info(String.format("setting up meta resource '%s' at '%s/%s'", name, metaAddress, metaCluster));
+ HelixAdmin admin = new ZKHelixAdmin(metaAddress);
+
+ log.info(String.format("setting up container '%s' (config='%s')", name, config));
+
+ admin.addResource(metaCluster, name, 1, "OnlineOffline", RebalanceMode.USER_DEFINED.toString());
+ IdealState idealState = admin.getResourceIdealState(metaCluster, name);
+ idealState.setRebalancerClassName(ProviderRebalancer.class.getName());
+ idealState.setReplicas("1");
+
+ // BEGIN workaround
+ // FIXME workaround for HELIX-226
+ Map<String, List<String>> listFields = Maps.newHashMap();
+ Map<String, Map<String, String>> mapFields = Maps.newHashMap();
+ for (int i = 0; i < 256; i++) {
+ String partitionName = name + "_" + i;
+ listFields.put(partitionName, new ArrayList<String>());
+ mapFields.put(partitionName, new HashMap<String, String>());
+ }
+ idealState.getRecord().setListFields(listFields);
+ idealState.getRecord().setMapFields(mapFields);
+ // END workaround
+
+ admin.setResourceIdealState(metaCluster, name, idealState);
+
+ HelixConfigScope scope = new HelixConfigScopeBuilder(ConfigScopeProperty.RESOURCE, metaCluster, name).build();
+ admin.setConfig(scope, this.config);
+
+ admin.close();
+ log.info("setup complete");
+ }
+
+ @Override
+ public void stop() throws Exception {
+ // left blank
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/MetaService.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/MetaService.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/MetaService.java
new file mode 100644
index 0000000..2e5e686
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/MetaService.java
@@ -0,0 +1,80 @@
+package org.apache.helix.metamanager.bootstrapper;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.helix.HelixAdmin;
+import org.apache.helix.manager.zk.ZKHelixAdmin;
+import org.apache.helix.metamanager.Service;
+import org.apache.helix.metamanager.bootstrap.BootUtils;
+import org.apache.helix.model.HelixConfigScope;
+import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
+import org.apache.helix.model.builder.HelixConfigScopeBuilder;
+import org.apache.log4j.Logger;
+
+import com.google.common.collect.Lists;
+
+public class MetaService implements Service {
+
+ static final Logger log = Logger.getLogger(MetaService.class);
+
+ Service service;
+
+ String clazz;
+ String metaAddress;
+
+ String metaCluster;
+
+ Properties config;
+
+ @Override
+ public void configure(Properties properties) throws Exception {
+ clazz = properties.getProperty("class");
+ metaAddress = properties.getProperty("metaaddress", "localhost:2199");
+ metaCluster = properties.getProperty("metacluster", "metacluster");
+
+ this.config = new Properties();
+ this.config.putAll(properties);
+ }
+
+ @Override
+ public void start() throws Exception {
+ log.info(String.format("starting service '%s' (config=%s)", clazz, config));
+
+ HelixAdmin admin = new ZKHelixAdmin(metaAddress);
+
+ HelixConfigScope managedScope = new HelixConfigScopeBuilder(ConfigScopeProperty.CLUSTER, metaCluster).build();
+ Map<String, String> managedProps = admin.getConfig(managedScope, Lists.newArrayList("cluster", "address"));
+ config.putAll(managedProps);
+
+ Collection<String> resources = admin.getResourcesInCluster(metaCluster);
+ config.put("containers", StringUtils.join(resources, ","));
+
+ for(String resource : admin.getResourcesInCluster(metaCluster)) {
+ HelixConfigScope resScope = new HelixConfigScopeBuilder(ConfigScopeProperty.RESOURCE, metaCluster, resource).build();
+ List<String> resKeys = admin.getConfigKeys(resScope);
+ Map<String, String> resProps = admin.getConfig(resScope, resKeys);
+
+ for(Map.Entry<String, String> entry : resProps.entrySet()) {
+ config.put(resource + "." + entry.getKey(), entry.getValue());
+ }
+ }
+
+ service = BootUtils.createInstance(clazz);
+ service.configure(config);
+ service.start();
+ }
+
+ @Override
+ public void stop() throws Exception {
+ log.info(String.format("stopping service '%s' (config=%s)", clazz, config));
+ if (service != null) {
+ service.stop();
+ service = null;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/ResourceService.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/ResourceService.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/ResourceService.java
new file mode 100644
index 0000000..35bed91
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/ResourceService.java
@@ -0,0 +1,61 @@
+package org.apache.helix.metamanager.bootstrapper;
+
+import java.util.Properties;
+
+import org.apache.helix.HelixAdmin;
+import org.apache.helix.manager.zk.ZKHelixAdmin;
+import org.apache.helix.metamanager.Service;
+import org.apache.helix.model.IdealState;
+import org.apache.helix.model.IdealState.RebalanceMode;
+import org.apache.log4j.Logger;
+
+/**
+ * Bootstrapping Helix resource. Create resource in Helix and configure
+ * properties.
+ *
+ */
+public class ResourceService implements Service {
+
+ static final Logger log = Logger.getLogger(ResourceService.class);
+
+ String cluster;
+ String address;
+ String container;
+ String name;
+ String model;
+ int partitions;
+ int replica;
+
+ @Override
+ public void configure(Properties properties) throws Exception {
+ cluster = properties.getProperty("cluster", "cluster");
+ address = properties.getProperty("address", "localhost:2199");
+ name = properties.getProperty("name", "resource");
+ container = properties.getProperty("container", "container");
+ model = properties.getProperty("model", "OnlineOffline");
+ partitions = Integer.parseInt(properties.getProperty("partitions", "1"));
+ replica = Integer.parseInt(properties.getProperty("replica", "1"));
+ }
+
+ @Override
+ public void start() throws Exception {
+ log.info(String.format("setting up resource '%s' at '%s/%s'", name, address, cluster));
+ HelixAdmin admin = new ZKHelixAdmin(address);
+
+ log.info(String.format("setting up resource '%s' (container='%s', model='%s', partitions=%d, replica=%d)", name, container, model, partitions, replica));
+
+ admin.addResource(cluster, name, partitions, model, RebalanceMode.FULL_AUTO.toString());
+ IdealState idealState = admin.getResourceIdealState(cluster, name);
+ idealState.setInstanceGroupTag(container);
+ idealState.setReplicas(String.valueOf(replica));
+ admin.setResourceIdealState(cluster, name, idealState);
+ admin.close();
+ log.info("setup complete");
+ }
+
+ @Override
+ public void stop() throws Exception {
+ // left blank
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/ZookeeperService.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/ZookeeperService.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/ZookeeperService.java
new file mode 100644
index 0000000..b220dc8
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/bootstrapper/ZookeeperService.java
@@ -0,0 +1,64 @@
+package org.apache.helix.metamanager.bootstrapper;
+
+import java.io.File;
+import java.util.Properties;
+
+import org.I0Itec.zkclient.IDefaultNameSpace;
+import org.I0Itec.zkclient.ZkClient;
+import org.I0Itec.zkclient.ZkServer;
+import org.apache.commons.io.FileUtils;
+import org.apache.helix.metamanager.Service;
+import org.apache.log4j.Logger;
+
+/**
+ * Bootstrapping zookeeper. Convenience tool for creating standalone zookeeper
+ * instance for test deployments. For production use a separate zookeeper
+ * cluster is strongly recommended.
+ *
+ */
+public class ZookeeperService implements Service {
+
+ static final Logger log = Logger.getLogger(ZookeeperService.class);
+
+ String dataDir;
+ String logDir;
+ int port;
+
+ ZkServer server;
+
+ @Override
+ public void configure(Properties properties) throws Exception {
+ dataDir = properties.getProperty("datadir", "/tmp/zk/data");
+ logDir = properties.getProperty("logdir", "/tmp/zk/log");
+ port = Integer.parseInt(properties.getProperty("port", "2199"));
+ }
+
+ @Override
+ public void start() {
+ log.info(String.format("starting zookeeper service (dataDir='%s', logDir='%s', port=%d)", dataDir, logDir, port));
+
+ FileUtils.deleteQuietly(new File(dataDir));
+ FileUtils.deleteQuietly(new File(logDir));
+
+ IDefaultNameSpace defaultNameSpace = new IDefaultNameSpace() {
+ @Override
+ public void createDefaultNameSpace(ZkClient zkClient) {
+ // left blank
+ }
+ };
+
+ server = new ZkServer(dataDir, logDir, defaultNameSpace, port);
+ server.start();
+ }
+
+ @Override
+ public void stop() {
+ log.info("stopping zookeeper service");
+
+ if (server != null) {
+ server.shutdown();
+ server = null;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/cluster/FileTargetProvider.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/cluster/FileTargetProvider.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/cluster/FileTargetProvider.java
new file mode 100644
index 0000000..b70d9ba
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/cluster/FileTargetProvider.java
@@ -0,0 +1,29 @@
+package org.apache.helix.metamanager.cluster;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.helix.metamanager.ClusterStatusProvider;
+
+
+public class FileTargetProvider implements ClusterStatusProvider {
+
+ final File file;
+
+ public FileTargetProvider(Properties properties) {
+ this.file = new File(properties.getProperty("path"));
+ }
+
+ @Override
+ public int getTargetContainerCount(String containerType) throws FileNotFoundException, IOException, IllegalArgumentException {
+ Properties properties = new Properties();
+ properties.load(new FileReader(file));
+ if(!properties.contains(containerType))
+ throw new IllegalArgumentException(String.format("container type '%s' not found in '%s'", containerType, file.getCanonicalPath()));
+ return Integer.parseInt((String)properties.get(containerType));
+ }
+
+}