You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by ki...@apache.org on 2013/09/20 20:30:15 UTC

[06/15] Adding Helix-task-framework and Yarn integration modules

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProcess.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProcess.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProcess.java
new file mode 100644
index 0000000..110fe68
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProcess.java
@@ -0,0 +1,64 @@
+package org.apache.helix.metamanager.impl.local;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.helix.HelixManager;
+import org.apache.helix.HelixManagerFactory;
+import org.apache.helix.InstanceType;
+import org.apache.helix.metamanager.container.ContainerStateModelFactory;
+import org.apache.log4j.Logger;
+
+public class LocalContainerProcess
+{
+  static final Logger log = Logger.getLogger(LocalContainerProcess.class);
+	
+  private String clusterName;
+  private String zkAddress;
+  private String instanceName;
+  private HelixManager participantManager;
+
+  public LocalContainerProcess(String clusterName, String zkAddress, String instanceName)
+  {
+    this.clusterName = clusterName;
+    this.zkAddress = zkAddress;
+    this.instanceName = instanceName;
+
+  }
+
+  public void start() throws Exception
+  {
+    log.info("STARTING "+ instanceName);
+    participantManager = HelixManagerFactory.getZKHelixManager(clusterName,
+        instanceName, InstanceType.PARTICIPANT, zkAddress);
+    participantManager.getStateMachineEngine().registerStateModelFactory(
+        "MasterSlave", new ContainerStateModelFactory());
+    participantManager.connect();
+    log.info("STARTED "+ instanceName);
+
+  }
+
+  public void stop()
+  {
+    if (participantManager != null)
+    {
+      participantManager.disconnect();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProvider.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProvider.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProvider.java
new file mode 100644
index 0000000..49c9d42
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProvider.java
@@ -0,0 +1,119 @@
+package org.apache.helix.metamanager.impl.local;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.helix.metamanager.ContainerProvider;
+import org.apache.helix.metamanager.ContainerProviderService;
+import org.apache.helix.metamanager.container.ContainerProcess;
+import org.apache.helix.metamanager.container.ContainerProcessProperties;
+import org.apache.helix.metamanager.container.ContainerUtils;
+import org.apache.helix.metamanager.impl.local.LocalContainerSingleton.LocalProcess;
+import org.apache.helix.metamanager.provider.ProviderProperties;
+import org.apache.log4j.Logger;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * {@link ContainerProvider} spawning VM-local containers. Only works in single-VM
+ * deployments as container metadata is managed via singleton.
+ * 
+ * @see LocalContainerSingleton
+ */
+class LocalContainerProvider implements ContainerProviderService {
+
+    static final Logger           log   = Logger.getLogger(LocalContainerProvider.class);
+
+    final Map<String, Properties> types = new HashMap<String, Properties>();
+
+    String                        address;
+    String                        cluster;
+    String                        name;
+
+    @Override
+    public void configure(Properties properties) throws Exception {
+        ProviderProperties providerProperties = new ProviderProperties();
+        providerProperties.putAll(properties);
+        Preconditions.checkArgument(providerProperties.isValid());
+
+        this.address = providerProperties.getProperty("address");
+        this.cluster = providerProperties.getProperty("cluster");
+        this.name = providerProperties.getProperty("name");
+
+        for (String containerType : providerProperties.getContainers()) {
+            registerType(containerType, providerProperties.getContainer(containerType));
+        }
+    }
+
+    @Override
+    public void start() throws Exception {
+        // left blank
+    }
+
+    @Override
+    public void stop() throws Exception {
+        destroyAll();
+    }
+
+    @Override
+    public void create(String id, String type) throws Exception {
+        Map<String, LocalProcess> processes = LocalContainerSingleton.getProcesses();
+
+        synchronized (processes) {
+            Preconditions.checkState(!processes.containsKey(id), "Process '%s' already exists", id);
+            Preconditions.checkState(types.containsKey(type), "Type '%s' is not registered", type);
+
+            ContainerProcessProperties properties = new ContainerProcessProperties(types.get(type));
+
+            properties.setProperty(ContainerProcessProperties.CLUSTER, cluster);
+            properties.setProperty(ContainerProcessProperties.NAME, id);
+            properties.setProperty(ContainerProcessProperties.ADDRESS, address);
+
+            log.info(String.format("Running container '%s' (properties='%s')", id, properties));
+
+            ContainerProcess process = ContainerUtils.createProcess(properties);
+            process.start();
+
+            processes.put(id, new LocalProcess(id, name, process));
+
+        }
+    }
+
+    @Override
+    public void destroy(String id) throws Exception {
+        Map<String, LocalProcess> processes = LocalContainerSingleton.getProcesses();
+
+        synchronized (processes) {
+            if (!processes.containsKey(id))
+                throw new IllegalArgumentException(String.format("Process '%s' does not exists", id));
+
+            log.info(String.format("Destroying container '%s'", id));
+
+            LocalProcess local = processes.remove(id);
+
+            local.process.stop();
+        }
+    }
+
+    @Override
+    public void destroyAll() {
+        Map<String, LocalProcess> processes = LocalContainerSingleton.getProcesses();
+
+        synchronized (processes) {
+            log.info("Destroying all processes");
+            for (LocalProcess local : new HashSet<LocalProcess>(processes.values())) {
+                if (local.owner.equals(name)) {
+                    try { destroy(local.id); } catch (Exception ignore) {}
+                }
+            }
+        }
+    }
+
+    void registerType(String name, Properties properties) {
+        log.debug(String.format("Registering container type '%s' (properties='%s')", name, properties));
+        types.put(name, properties);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProviderProcess.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProviderProcess.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProviderProcess.java
new file mode 100644
index 0000000..ed090bc
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProviderProcess.java
@@ -0,0 +1,45 @@
+package org.apache.helix.metamanager.impl.local;
+
+import java.util.Properties;
+
+import org.apache.helix.metamanager.Service;
+import org.apache.helix.metamanager.provider.ProviderProcess;
+import org.apache.helix.metamanager.provider.ProviderProperties;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Configurable and runnable service for {@link LocalContainerProvider}. 
+ *
+ */
+public class LocalContainerProviderProcess implements Service {
+    LocalContainerProvider provider;
+    ProviderProcess        process;
+
+    @Override
+    public void configure(Properties properties) throws Exception {
+        ProviderProperties providerProperties = new ProviderProperties();
+        providerProperties.putAll(properties);
+        
+        Preconditions.checkArgument(providerProperties.isValid(), "provider properties not valid (properties='%s')", properties);
+        
+        provider = new LocalContainerProvider();
+        provider.configure(properties);
+
+        process = new ProviderProcess();
+        process.configure(providerProperties);
+        process.setConteinerProvider(provider);
+    }
+
+    @Override
+    public void start() throws Exception {
+        provider.start();
+        process.start();
+    }
+
+    @Override
+    public void stop() throws Exception {
+        process.stop();
+        provider.stop();
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerSingleton.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerSingleton.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerSingleton.java
new file mode 100644
index 0000000..b91a848
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerSingleton.java
@@ -0,0 +1,56 @@
+package org.apache.helix.metamanager.impl.local;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.helix.metamanager.container.ContainerProcess;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Singleton tracking metadata for VM-local containers spawned via
+ * {@link LocalContainerProvider}.
+ * 
+ */
+public class LocalContainerSingleton {
+    final static Map<String, LocalProcess> processes = new HashMap<String, LocalProcess>();
+
+    private LocalContainerSingleton() {
+        // left blank
+    }
+
+    public static Map<String, LocalProcess> getProcesses() {
+        return processes;
+    }
+
+    public static void reset() {
+        synchronized (processes) {
+            for (LocalProcess local : processes.values()) {
+                local.process.stop();
+            }
+            processes.clear();
+        }
+    }
+
+    public static void killProcess(String id) throws InterruptedException {
+        synchronized (processes) {
+            Preconditions.checkArgument(processes.containsKey(id), "Process '%s' does not exist", id);
+            ContainerProcess process = processes.get(id).process;
+            process.stop();
+            processes.remove(id);
+        }
+    }
+
+    static class LocalProcess {
+        final String           id;
+        final String           owner;
+        final ContainerProcess process;
+
+        public LocalProcess(String id, String owner, ContainerProcess process) {
+            this.id = id;
+            this.owner = owner;
+            this.process = process;
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerStatusProvider.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerStatusProvider.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerStatusProvider.java
new file mode 100644
index 0000000..604ad25
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerStatusProvider.java
@@ -0,0 +1,37 @@
+package org.apache.helix.metamanager.impl.local;
+
+import java.util.Map;
+
+import org.apache.helix.metamanager.ContainerStatusProvider;
+import org.apache.helix.metamanager.impl.local.LocalContainerSingleton.LocalProcess;
+
+public class LocalContainerStatusProvider implements ContainerStatusProvider {
+
+	@Override
+	public boolean exists(String id) {
+		Map<String, LocalProcess> processes = LocalContainerSingleton.getProcesses();
+		
+		synchronized (processes) {
+			return processes.containsKey(id);
+		}
+	}
+
+	@Override
+	public boolean isActive(String id) {
+		Map<String, LocalProcess> processes = LocalContainerSingleton.getProcesses();
+		
+		synchronized (processes) {
+			return processes.get(id).process.isActive();
+		}
+	}
+
+	@Override
+	public boolean isFailed(String id) {
+		Map<String, LocalProcess> processes = LocalContainerSingleton.getProcesses();
+		
+		synchronized (processes) {
+			return processes.get(id).process.isFailed();
+		}
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalStatusProvider.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalStatusProvider.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalStatusProvider.java
new file mode 100644
index 0000000..c53a3ba
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalStatusProvider.java
@@ -0,0 +1,53 @@
+package org.apache.helix.metamanager.impl.local;
+
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.helix.metamanager.StatusProviderService;
+import org.apache.helix.metamanager.impl.local.LocalContainerSingleton.LocalProcess;
+
+/**
+ * StatusProvider for VM-local containers spawned via
+ * {@link LocalContainerProvider}. Runnable and configurable service.
+ * 
+ */
+public class LocalStatusProvider implements StatusProviderService {
+
+    @Override
+    public boolean exists(String id) {
+        Map<String, LocalProcess> processes = LocalContainerSingleton.getProcesses();
+
+        synchronized (processes) {
+            return processes.containsKey(id);
+        }
+    }
+
+    @Override
+    public boolean isHealthy(String id) {
+        Map<String, LocalProcess> processes = LocalContainerSingleton.getProcesses();
+
+        synchronized (processes) {
+            LocalProcess local = processes.get(id);
+
+            if (local == null)
+                return false;
+
+            return local.process.isActive();
+        }
+    }
+
+    @Override
+    public void configure(Properties properties) throws Exception {
+        // left blank
+    }
+
+    @Override
+    public void start() throws Exception {
+        // left blank
+    }
+
+    @Override
+    public void stop() throws Exception {
+        // left blank
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProcess.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProcess.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProcess.java
new file mode 100644
index 0000000..0069110
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProcess.java
@@ -0,0 +1,93 @@
+package org.apache.helix.metamanager.impl.shell;
+
+import java.io.File;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.helix.metamanager.container.ContainerProcess;
+import org.apache.helix.metamanager.container.ContainerProcessProperties;
+import org.apache.helix.metamanager.container.ContainerUtils;
+import org.apache.log4j.Logger;
+
+/**
+ * Host process for Shell-based container. ContainerProcess configuration is
+ * read from path in first command-line argument. Status is maintained using
+ * temporary marker file. (Program entry point)
+ * 
+ */
+class ShellContainerProcess {
+    static final Logger             log              = Logger.getLogger(ShellContainerProcess.class);
+
+    public static final long        MONITOR_INTERVAL = 5000;
+
+    static String                   markerDir;
+    static ContainerProcess         process;
+    static ScheduledExecutorService executor         = Executors.newSingleThreadScheduledExecutor();
+
+    public static void main(String[] args) throws Exception {
+        final String propertiesPath = args[0];
+        markerDir = args[1];
+
+        ContainerProcessProperties properties = ContainerUtils.getPropertiesFromPath(propertiesPath);
+
+        process = ContainerUtils.createProcess(properties);
+
+        log.debug("Installing shutdown hooks");
+        Runtime.getRuntime().addShutdownHook(new Thread() {
+            @Override
+            public void run() {
+                log.debug("Running shutdown hook");
+                try {
+                    ShellContainerProcess.stop();
+                } catch (Exception ignore) {
+                }
+            }
+        });
+
+        log.debug("Launching shell container process");
+        process.start();
+
+        ShellUtils.createMarker(new File(markerDir));
+
+        log.debug("Launching process monitor");
+        executor.scheduleAtFixedRate(new ProcessMonitor(), 0, MONITOR_INTERVAL, TimeUnit.MILLISECONDS);
+    }
+
+    static void stop() throws InterruptedException {
+        log.debug("Shutting down shell process");
+        if (process != null) {
+            process.stop();
+            ShellUtils.destroyMarker(new File(markerDir));
+        }
+        if (executor != null) {
+            executor.shutdownNow();
+            while (!executor.isTerminated()) {
+                Thread.sleep(100);
+            }
+            executor = null;
+        }
+    }
+
+    static class ProcessMonitor implements Runnable {
+        @Override
+        public void run() {
+            if (process.isFailed()) {
+                log.warn("detected process failure");
+                try {
+                    ShellContainerProcess.stop();
+                } catch (Exception ignore) {
+                }
+                System.exit(1);
+            }
+            if (!process.isActive()) {
+                log.warn("detected process shutdown");
+                try {
+                    ShellContainerProcess.stop();
+                } catch (Exception ignore) {
+                }
+            }
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProvider.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProvider.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProvider.java
new file mode 100644
index 0000000..69e2553
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProvider.java
@@ -0,0 +1,151 @@
+package org.apache.helix.metamanager.impl.shell;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.helix.metamanager.ContainerProvider;
+import org.apache.helix.metamanager.ContainerProviderService;
+import org.apache.helix.metamanager.container.ContainerProcessProperties;
+import org.apache.helix.metamanager.impl.shell.ShellContainerSingleton.ShellProcess;
+import org.apache.helix.metamanager.provider.ProviderProperties;
+import org.apache.log4j.Logger;
+
+import com.google.common.base.Preconditions;
+import com.google.common.io.Files;
+
+/**
+ * {@link ContainerProvider} spawning shell-based containers. Only works in single-VM
+ * deployments as container metadata is managed via singleton.
+ * 
+ * @see ShellContainerSingleton
+ */
+class ShellContainerProvider implements ContainerProviderService {
+
+    static final Logger                    log               = Logger.getLogger(ShellContainerProvider.class);
+
+    static final String                    RUN_COMMAND       = "/bin/sh";
+
+    static final long                      POLL_INTERVAL     = 1000;
+    static final long                      CONTAINER_TIMEOUT = 60000;
+
+    // global view of processes required
+    static final Map<String, ShellProcess> processes         = new HashMap<String, ShellProcess>();
+
+    final Map<String, Properties>          types             = new HashMap<String, Properties>();
+
+    String                                 address;
+    String                                 cluster;
+    String                                 name;
+
+    @Override
+    public void configure(Properties properties) throws Exception {
+        Preconditions.checkNotNull(properties);
+        ProviderProperties providerProperties = new ProviderProperties();
+        providerProperties.putAll(properties);
+        Preconditions.checkArgument(providerProperties.isValid());
+
+        this.address = providerProperties.getProperty("address");
+        this.cluster = providerProperties.getProperty("cluster");
+        this.name = providerProperties.getProperty("name");
+
+        for (String containerType : providerProperties.getContainers()) {
+            registerType(containerType, providerProperties.getContainer(containerType));
+        }
+    }
+
+    @Override
+    public void start() throws Exception {
+        // left blank
+    }
+
+    @Override
+    public void stop() throws Exception {
+        destroyAll();
+    }
+
+    @Override
+    public void create(String id, String type) throws Exception {
+        Map<String, ShellProcess> processes = ShellContainerSingleton.getProcesses();
+
+        synchronized (processes) {
+            Preconditions.checkState(!processes.containsKey(id), "Process '%s' already exists", id);
+            Preconditions.checkState(types.containsKey(type), "Type '%s' is not registered", type);
+
+            ContainerProcessProperties properties = new ContainerProcessProperties(types.get(type));
+
+            properties.setProperty(ContainerProcessProperties.CLUSTER, cluster);
+            properties.setProperty(ContainerProcessProperties.NAME, id);
+            properties.setProperty(ContainerProcessProperties.ADDRESS, address);
+
+            File tmpDir = Files.createTempDir();
+            File tmpProperties = new File(tmpDir.getCanonicalPath() + File.separator + ShellUtils.SHELL_CONTAINER_PROPERTIES);
+            File tmpMarker = new File(tmpDir.getCanonicalPath());
+
+            properties.store(new FileWriter(tmpProperties), id);
+
+            log.info(String.format("Running container '%s' (properties='%s')", id, properties));
+
+            log.debug(String.format("Invoking command '%s %s %s %s'", RUN_COMMAND, ShellUtils.SHELL_CONTAINER_PATH, tmpProperties.getCanonicalPath(),
+                    tmpMarker.getCanonicalPath()));
+
+            ProcessBuilder builder = new ProcessBuilder();
+            builder.command(RUN_COMMAND, ShellUtils.SHELL_CONTAINER_PATH, tmpProperties.getCanonicalPath(), tmpMarker.getCanonicalPath());
+
+            Process process = builder.start();
+
+            processes.put(id, new ShellProcess(id, name, process, tmpDir));
+
+            long limit = System.currentTimeMillis() + CONTAINER_TIMEOUT;
+            while (!ShellUtils.hasMarker(tmpDir)) {
+                if (System.currentTimeMillis() >= limit) {
+                    throw new TimeoutException(String.format("Container '%s' failed to reach active state", id));
+                }
+                Thread.sleep(POLL_INTERVAL);
+            }
+        }
+    }
+
+    @Override
+    public void destroy(String id) throws Exception {
+        Map<String, ShellProcess> processes = ShellContainerSingleton.getProcesses();
+
+        synchronized (processes) {
+            if (!processes.containsKey(id))
+                throw new IllegalArgumentException(String.format("Process '%s' does not exists", id));
+
+            log.info(String.format("Destroying container '%s'", id));
+
+            ShellProcess shell = processes.remove(id);
+            shell.process.destroy();
+            shell.process.waitFor();
+
+            FileUtils.deleteDirectory(shell.tmpDir);
+        }
+    }
+
+    @Override
+    public void destroyAll() {
+        Map<String, ShellProcess> processes = ShellContainerSingleton.getProcesses();
+
+        synchronized (processes) {
+            log.info("Destroying all owned processes");
+            for (ShellProcess process : new HashSet<ShellProcess>(processes.values())) {
+                if (process.owner.equals(name)) {
+			        try { destroy(process.id); } catch (Exception ignore) {}
+                }
+            }
+        }
+    }
+
+    void registerType(String name, Properties properties) {
+        log.debug(String.format("Registering container type '%s' (properties='%s')", name, properties));
+        types.put(name, properties);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProviderProcess.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProviderProcess.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProviderProcess.java
new file mode 100644
index 0000000..36f150d
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProviderProcess.java
@@ -0,0 +1,45 @@
+package org.apache.helix.metamanager.impl.shell;
+
+import java.util.Properties;
+
+import org.apache.helix.metamanager.Service;
+import org.apache.helix.metamanager.provider.ProviderProcess;
+import org.apache.helix.metamanager.provider.ProviderProperties;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Configurable and runnable service for {@link ShellContainerProvider}.
+ * 
+ */
+public class ShellContainerProviderProcess implements Service {
+    ShellContainerProvider provider;
+    ProviderProcess        process;
+
+    @Override
+    public void configure(Properties properties) throws Exception {
+        ProviderProperties providerProperties = new ProviderProperties();
+        providerProperties.putAll(properties);
+
+        Preconditions.checkArgument(providerProperties.isValid(), "provider properties not valid (properties='%s')", properties);
+
+        provider = new ShellContainerProvider();
+        provider.configure(properties);
+
+        process = new ProviderProcess();
+        process.configure(providerProperties);
+        process.setConteinerProvider(provider);
+    }
+
+    @Override
+    public void start() throws Exception {
+        provider.start();
+        process.start();
+    }
+
+    @Override
+    public void stop() throws Exception {
+        process.stop();
+        provider.stop();
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerSingleton.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerSingleton.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerSingleton.java
new file mode 100644
index 0000000..91054e1
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerSingleton.java
@@ -0,0 +1,58 @@
+package org.apache.helix.metamanager.impl.shell;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Singleton tracking metadata for shell-based containers spawned via
+ * {@link ShellContainerProvider}.
+ * 
+ */
+public class ShellContainerSingleton {
+    static final Map<String, ShellProcess> processes = new HashMap<String, ShellProcess>();
+
+    private ShellContainerSingleton() {
+        // left blank
+    }
+
+    public static Map<String, ShellProcess> getProcesses() {
+        return processes;
+    }
+
+    public static void reset() {
+        synchronized (processes) {
+            for (ShellProcess shell : processes.values()) {
+                shell.process.destroy();
+				try { shell.process.waitFor(); } catch(Exception ignore) {}
+            }
+            processes.clear();
+        }
+    }
+
+    public static void killProcess(String id) throws InterruptedException {
+        synchronized (processes) {
+            Preconditions.checkArgument(processes.containsKey(id), "Process '%s' does not exist", id);
+            Process process = processes.get(id).process;
+            process.destroy();
+            process.waitFor();
+            processes.remove(id);
+        }
+    }
+
+    static class ShellProcess {
+        final String  id;
+        final String  owner;
+        final Process process;
+        final File    tmpDir;
+
+        public ShellProcess(String id, String owner, Process process, File tmpDir) {
+            this.id = id;
+            this.owner = owner;
+            this.process = process;
+            this.tmpDir = tmpDir;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerStatusProvider.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerStatusProvider.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerStatusProvider.java
new file mode 100644
index 0000000..03e55c6
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerStatusProvider.java
@@ -0,0 +1,52 @@
+package org.apache.helix.metamanager.impl.shell;
+
+import java.util.Map;
+
+import org.apache.helix.metamanager.ContainerStatusProvider;
+import org.apache.helix.metamanager.impl.shell.ShellContainerSingleton.ShellProcess;
+
+public class ShellContainerStatusProvider implements ContainerStatusProvider {
+
+	@Override
+	public boolean exists(String id) {
+		Map<String, ShellProcess> processes = ShellContainerSingleton.getProcesses();
+
+		synchronized (processes) {
+			return processes.containsKey(id);
+		}
+	}
+
+	@Override
+	public boolean isActive(String id) {
+		Map<String, ShellProcess> processes = ShellContainerSingleton.getProcesses();
+
+		synchronized (processes) {
+			ShellProcess shell = processes.get(id);
+			
+			try {
+				shell.process.exitValue();
+				return false;
+			} catch (IllegalThreadStateException e) {
+				// still running
+				return true;
+			}
+		}
+	}
+
+	@Override
+	public boolean isFailed(String id) {
+		Map<String, ShellProcess> processes = ShellContainerSingleton.getProcesses();
+
+		synchronized (processes) {
+			ShellProcess shell = processes.get(id);
+			
+			try {
+				return (shell.process.exitValue() != 0);
+			} catch (IllegalThreadStateException e) {
+				// still running
+				return false;
+			}
+		}
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellStatusProvider.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellStatusProvider.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellStatusProvider.java
new file mode 100644
index 0000000..015218b
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellStatusProvider.java
@@ -0,0 +1,64 @@
+package org.apache.helix.metamanager.impl.shell;
+
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.helix.metamanager.StatusProviderService;
+import org.apache.helix.metamanager.impl.shell.ShellContainerSingleton.ShellProcess;
+
+/**
+ * StatusProvider for shell-based containers spawned via
+ * {@link ShellContainerProvider}. Runnable and configurable service.
+ * 
+ */
+public class ShellStatusProvider implements StatusProviderService {
+
+    @Override
+    public boolean exists(String id) {
+        Map<String, ShellProcess> processes = ShellContainerSingleton.getProcesses();
+
+        synchronized (processes) {
+            return processes.containsKey(id);
+        }
+    }
+
+    @Override
+    public boolean isHealthy(String id) {
+        Map<String, ShellProcess> processes = ShellContainerSingleton.getProcesses();
+
+        synchronized (processes) {
+            ShellProcess shell = processes.get(id);
+
+            if (shell == null)
+                return false;
+
+            if (!ShellUtils.hasMarker(shell.tmpDir))
+                return false;
+
+            try {
+                // exit value
+                shell.process.exitValue();
+                return false;
+            } catch (IllegalThreadStateException e) {
+                // expected
+            }
+
+            return true;
+        }
+    }
+
+    @Override
+    public void configure(Properties properties) throws Exception {
+        // left blank
+    }
+
+    @Override
+    public void start() throws Exception {
+        // left blank
+    }
+
+    @Override
+    public void stop() throws Exception {
+        // left blank
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellUtils.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellUtils.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellUtils.java
new file mode 100644
index 0000000..dcec2ae
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellUtils.java
@@ -0,0 +1,54 @@
+package org.apache.helix.metamanager.impl.shell;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.log4j.Logger;
+
+/**
+ * Utility for creating and destroying temporary marker files for shell-based
+ * containers.
+ * 
+ */
+class ShellUtils {
+
+    static final Logger log                        = Logger.getLogger(ShellUtils.class);
+
+    static final String SHELL_CONTAINER_PATH       = "target/metamanager-pkg/bin/shell-container-process.sh";
+    static final String SHELL_CONTAINER_PROPERTIES = "container.properties";
+    static final String SHELL_CONTAINER_MARKER     = "active";
+
+    private ShellUtils() {
+        // left blank
+    }
+
+    public static boolean hasMarker(File processDir) {
+        try {
+            log.debug(String.format("checking for marker file '%s'", getMarkerFile(processDir)));
+            if (getMarkerFile(processDir).exists())
+                return true;
+        } catch (IOException e) {
+            // ignore
+        }
+        return false;
+    }
+
+    public static void createMarker(File processDir) throws IOException {
+        log.debug(String.format("creating marker file '%s'", getMarkerFile(processDir)));
+        getMarkerFile(processDir).createNewFile();
+    }
+
+    public static void destroyMarker(File processDir) {
+        try {
+            log.debug(String.format("destroying marker file '%s'", getMarkerFile(processDir)));
+            getMarkerFile(processDir).delete();
+        } catch (IOException e) {
+            // ignore
+        }
+    }
+
+    public static File getMarkerFile(File processDir) throws IOException {
+        return new File(processDir.getCanonicalPath() + File.separatorChar + SHELL_CONTAINER_MARKER);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/ApplicationConfig.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/ApplicationConfig.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/ApplicationConfig.java
new file mode 100644
index 0000000..9c85232
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/ApplicationConfig.java
@@ -0,0 +1,32 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+public class ApplicationConfig {
+	final String clusterAddress;
+	final String clusterName;
+	final String metadataAddress;
+	final String providerName;
+
+	public ApplicationConfig(String clusterAddress, String clusterName,
+			String metadataAddress, String providerName) {
+		this.clusterAddress = clusterAddress;
+		this.clusterName = clusterName;
+		this.metadataAddress = metadataAddress;
+		this.providerName = providerName;
+	}
+
+	public String getClusterAddress() {
+		return clusterAddress;
+	}
+
+	public String getClusterName() {
+		return clusterName;
+	}
+
+	public String getMetadataAddress() {
+		return metadataAddress;
+	}
+
+	public String getProviderName() {
+		return providerName;
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/ContainerMetadata.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/ContainerMetadata.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/ContainerMetadata.java
new file mode 100644
index 0000000..7b25d31
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/ContainerMetadata.java
@@ -0,0 +1,80 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+
+class ContainerMetadata {
+
+	static enum ContainerState {
+		ACQUIRE,
+		CONNECTING,
+		ACTIVE,
+		TEARDOWN,
+		FAILED,
+		HALTED,
+		FINALIZE
+	}
+	
+	String id;
+	ContainerState state;
+	int yarnId;
+	String owner;
+	YarnContainerProcessProperties properties;
+
+	public ContainerMetadata() {
+		// left blank
+	}
+	
+	public ContainerMetadata(String id, String owner, YarnContainerProcessProperties properties) {
+		this.id = id;
+		this.state = ContainerState.ACQUIRE;
+		this.yarnId = -1;
+		this.owner = owner;
+		this.properties = properties;
+	}
+
+	public String getId() {
+		return id;
+	}
+
+	public ContainerMetadata setId(String id) {
+		this.id = id;
+		return this;
+	}
+
+	public ContainerState getState() {
+		return state;
+	}
+
+	public ContainerMetadata setState(ContainerState state) {
+		this.state = state;
+		return this;
+	}
+
+	public int getYarnId() {
+		return yarnId;
+	}
+
+	public ContainerMetadata setYarnId(int yarnId) {
+		this.yarnId = yarnId;
+		return this;
+	}
+
+	public String getOwner() {
+		return owner;
+	}
+
+	public ContainerMetadata setOwner(String owner) {
+		this.owner = owner;
+		return this;
+	}
+
+	public YarnContainerProcessProperties getProperties() {
+		return properties;
+	}
+
+	public ContainerMetadata setProperties(YarnContainerProcessProperties properties) {
+		this.properties = properties;
+		return this;
+	}
+	
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/MetadataProvider.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/MetadataProvider.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/MetadataProvider.java
new file mode 100644
index 0000000..a35c16e
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/MetadataProvider.java
@@ -0,0 +1,42 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+import java.util.Collection;
+
+interface MetadataProvider {
+
+	public boolean exists(String id);
+
+	public void create(ContainerMetadata meta) throws MetadataException;
+
+	public ContainerMetadata read(String id) throws MetadataException;
+
+	public Collection<ContainerMetadata> readAll() throws MetadataException;
+
+	public void update(ContainerMetadata meta) throws MetadataException;
+
+	public void delete(String id) throws MetadataException;
+	
+	public static class MetadataException extends Exception {
+
+		/**
+		 * 
+		 */
+		private static final long serialVersionUID = -2846997013918977056L;
+
+		public MetadataException() {
+			super();
+		}
+
+		public MetadataException(String message, Throwable cause) {
+			super(message, cause);
+		}
+
+		public MetadataException(String message) {
+			super(message);
+		}
+
+		public MetadataException(Throwable cause) {
+			super(cause);
+		}	
+	}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/MetadataService.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/MetadataService.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/MetadataService.java
new file mode 100644
index 0000000..35dd1ee
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/MetadataService.java
@@ -0,0 +1,42 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+import java.util.Collection;
+
+public interface MetadataService {
+
+	public boolean exists(String id);
+
+	public void create(ContainerMetadata meta) throws MetadataServiceException;
+
+	public ContainerMetadata read(String id) throws MetadataServiceException;
+
+	public Collection<ContainerMetadata> readAll() throws MetadataServiceException;
+
+	public void update(ContainerMetadata meta) throws MetadataServiceException;
+
+	public void delete(String id) throws MetadataServiceException;
+	
+	public static class MetadataServiceException extends Exception {
+
+		/**
+		 * 
+		 */
+		private static final long serialVersionUID = -2846997013918977056L;
+
+		public MetadataServiceException() {
+			super();
+		}
+
+		public MetadataServiceException(String message, Throwable cause) {
+			super(message, cause);
+		}
+
+		public MetadataServiceException(String message) {
+			super(message);
+		}
+
+		public MetadataServiceException(Throwable cause) {
+			super(cause);
+		}	
+	}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/Utils.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/Utils.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/Utils.java
new file mode 100644
index 0000000..99f9a03
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/Utils.java
@@ -0,0 +1,94 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.helix.metamanager.impl.yarn.ContainerMetadata.ContainerState;
+import org.apache.log4j.Logger;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.TypeAdapter;
+import com.google.gson.stream.JsonReader;
+import com.google.gson.stream.JsonToken;
+import com.google.gson.stream.JsonWriter;
+
+public class Utils {
+	
+	static final Logger log = Logger.getLogger(Utils.class);
+	
+	static Gson gson;
+	static {
+		GsonBuilder builder = new GsonBuilder();
+		builder.registerTypeAdapter(ContainerState.class, new ContainerStateAdapter());
+		builder.setPrettyPrinting();
+		gson = builder.create();
+	}
+	
+	static Map<String, LocalResource>  dummyResources = createDummyResources();
+	
+	static String toJson(ContainerMetadata meta) {
+		return gson.toJson(meta);
+	}
+	
+	static ContainerMetadata fromJson(String json) {
+		return gson.fromJson(json, ContainerMetadata.class);
+	}
+	
+	static Map<String, LocalResource> getDummyResources() {
+		return dummyResources;
+	}
+
+	private static Map<String, LocalResource> createDummyResources() {
+		File dummy = new File("/tmp/dummy");
+		
+		if(!dummy.exists()) {
+	    	try {
+	    		dummy.createNewFile();
+	    	} catch(Exception e) {
+	    		log.error("could not create dummy file", e);
+	    		System.exit(1);
+	    	}
+		}
+	    
+	    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
+	    Path path = new Path(dummy.toURI());
+	    LocalResource localResource = Records.newRecord(LocalResource.class);
+	    localResource.setType(LocalResourceType.FILE);
+	    localResource.setVisibility(LocalResourceVisibility.APPLICATION);          
+	    localResource.setResource(ConverterUtils.getYarnUrlFromPath(path)); 
+	    localResource.setTimestamp(dummy.lastModified());
+	    localResource.setSize(dummy.length());
+	    localResources.put("dummy", localResource);
+		return localResources;
+	}
+	
+	static class ContainerStateAdapter extends TypeAdapter<ContainerState> {
+		@Override
+		public ContainerState read(JsonReader reader) throws IOException {
+			if (reader.peek() == JsonToken.NULL) {
+				reader.nextNull();
+				return null;
+			}
+			return ContainerState.valueOf(reader.nextString());
+		}
+
+		@Override
+		public void write(JsonWriter writer, ContainerState value) throws IOException {
+			if (value == null) {
+				writer.nullValue();
+				return;
+			}
+			writer.value(value.name());
+		}
+	}
+	
+}

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnApplication.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnApplication.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnApplication.java
new file mode 100644
index 0000000..3b7dcd6
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnApplication.java
@@ -0,0 +1,171 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+import java.io.File;
+import java.net.InetSocketAddress;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.ClientRMProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.helix.metamanager.ConfigTool;
+import org.apache.helix.metamanager.Service;
+import org.apache.log4j.Logger;
+
+class YarnApplication implements Service {
+
+	static final Logger log = Logger.getLogger(YarnApplication.class);
+	
+	static final String ENV_CLUSTER_ADDRESS  = "YA_CLUSTER_ADDRESS";
+	static final String ENV_CLUSTER_NAME     = "YA_CLUSTER_NAME";
+	static final String ENV_METADATA_ADDRESS = "YA_METADATA_ADDRESS";
+	static final String ENV_PROVIDER_NAME    = "YA_PROVIDER_NAME";
+
+	static String YARN_MASTER_COMMAND = "/bin/sh %s 1>%s/stdout 2>%s/stderr";
+	
+	Configuration conf;
+	YarnRPC rpc;
+	ClientRMProtocol rmClient;
+	ApplicationId appId;
+	File propertiesFile;
+	
+	YarnApplicationProperties properties;
+
+	public YarnApplication() {
+	    // left blank
+	}
+	
+    public YarnApplication(YarnApplicationProperties properties) {
+        this.properties = properties;
+        internalConf();
+    }
+    
+    @Override
+    public void configure(Properties properties) throws Exception {
+        YarnApplicationProperties yarnProps = new YarnApplicationProperties();
+        yarnProps.putAll(properties);
+        this.properties = yarnProps;
+        internalConf();
+    }
+
+    public void internalConf() {
+        this.conf = new YarnConfiguration();
+        this.conf.set(YarnConfiguration.RM_ADDRESS, properties.getYarnResourceManager());
+        this.conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, properties.getYarnScheduler());
+        this.conf.set(FileSystem.FS_DEFAULT_NAME_KEY, properties.getYarnHdfs());
+
+        this.rpc = YarnRPC.create(conf);
+    }
+    
+    @Override
+    public void start() throws Exception {
+		connect();
+		
+		String command = String.format(YARN_MASTER_COMMAND, ConfigTool.YARN_MASTER_PATH,
+				ApplicationConstants.LOG_DIR_EXPANSION_VAR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+
+		log.info(String.format("Starting application '%s' provider '%s' (masterCommand='%s')",
+				properties.getProviderMetadata(), properties.getProviderName(), command));
+
+		log.debug(String.format("Running master command \"%s\"", command));
+		
+		// app id
+		GetNewApplicationRequest appRequest = Records.newRecord(GetNewApplicationRequest.class);
+		GetNewApplicationResponse appResponse = rmClient.getNewApplication(appRequest);
+
+		this.appId = appResponse.getApplicationId();
+
+		log.info(String.format("Acquired app id '%s' for '%s'", appId.toString(), properties.getProviderName()));
+		
+		// command
+		ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class);
+		launchContext.setCommands(Collections.singletonList(command));
+
+		// resource limit
+		Resource resource = Records.newRecord(Resource.class);
+		resource.setMemory(256); // TODO make dynamic
+		launchContext.setResource(resource);
+		
+	    // environment
+	    Map<String, String> env = new HashMap<String, String>();
+	    launchContext.setEnvironment(env);
+	    
+	    // configuration
+	    propertiesFile = YarnUtils.writePropertiesToTemp(properties);
+	    
+	    // HDFS
+	    final String namespace = appId.toString();
+	    final Path masterArchive = YarnUtils.copyToHdfs(ConfigTool.YARN_MASTER_ARCHIVE_PATH, ConfigTool.YARN_MASTER_STAGING, namespace, conf);
+	    final Path masterProperties = YarnUtils.copyToHdfs(propertiesFile.getCanonicalPath(), ConfigTool.YARN_MASTER_PROPERTIES, namespace, conf);
+	    final Path containerArchive = YarnUtils.copyToHdfs(ConfigTool.YARN_CONTAINER_ARCHIVE_PATH, ConfigTool.YARN_CONTAINER_STAGING, namespace, conf);
+	    
+	    // local resources
+	    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
+	    localResources.put(ConfigTool.YARN_MASTER_DESTINATION, 
+	    		YarnUtils.createHdfsResource(masterArchive, LocalResourceType.ARCHIVE, conf));
+	    localResources.put(ConfigTool.YARN_MASTER_PROPERTIES,
+	    		YarnUtils.createHdfsResource(masterProperties, LocalResourceType.FILE, conf));
+        localResources.put(ConfigTool.YARN_CONTAINER_STAGING,
+                YarnUtils.createHdfsResource(containerArchive, LocalResourceType.FILE, conf));
+	    
+	    launchContext.setLocalResources(localResources);
+	    
+	    // user
+	    launchContext.setUser(properties.getYarnUser());
+	    
+	    // app submission
+	    ApplicationSubmissionContext subContext = Records.newRecord(ApplicationSubmissionContext.class);
+		subContext.setApplicationId(appId);
+		subContext.setApplicationName(properties.getProviderName());
+		subContext.setAMContainerSpec(launchContext);
+
+		SubmitApplicationRequest subRequest = Records.newRecord(SubmitApplicationRequest.class);
+		subRequest.setApplicationSubmissionContext(subContext);
+		
+		log.info(String.format("Starting app id '%s'", appId.toString()));
+
+		rmClient.submitApplication(subRequest);
+		
+	}
+
+    @Override
+	public void stop() throws YarnRemoteException {
+		log.info(String.format("Stopping app id '%s'", appId.toString()));
+		KillApplicationRequest killRequest = Records.newRecord(KillApplicationRequest.class);
+		killRequest.setApplicationId(appId);
+
+		rmClient.forceKillApplication(killRequest);
+
+		try { YarnUtils.destroyHdfsNamespace(appId.toString(), conf); } catch(Exception ignore) {}
+		
+		propertiesFile.delete();
+	}
+
+	void connect() {
+		YarnConfiguration yarnConf = new YarnConfiguration(conf);
+		InetSocketAddress rmAddress = NetUtils.createSocketAddr(yarnConf.get(
+				YarnConfiguration.RM_ADDRESS,
+				YarnConfiguration.DEFAULT_RM_ADDRESS));
+		log.info("Connecting to ResourceManager at: " + rmAddress);
+		this.rmClient = ((ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class, rmAddress, conf));
+	}
+}

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnApplicationProperties.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnApplicationProperties.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnApplicationProperties.java
new file mode 100644
index 0000000..e047179
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnApplicationProperties.java
@@ -0,0 +1,91 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+import java.util.Properties;
+
+import org.apache.helix.metamanager.container.ContainerProcessProperties;
+
+import com.google.common.base.Preconditions;
+
+public class YarnApplicationProperties extends Properties {
+	/**
+	 * 
+	 */
+	private static final long serialVersionUID = -2209509977839674160L;
+	
+	public final static String HELIX_ZOOKEEPER = ContainerProcessProperties.HELIX_ZOOKEEPER;
+	public final static String HELIX_CLUSTER = ContainerProcessProperties.HELIX_CLUSTER;
+	public final static String PROVIDER_METADATA = "provider.metadata";
+	public final static String PROVIDER_NAME = "provider.name";
+	public final static String CONTAINER_ID = "container.id";
+    public final static String YARN_RESOURCEMANAGER = "yarn.resourcemananger";
+    public final static String YARN_SCHEDULER = "yarn.scheduler";
+    public final static String YARN_USER = "yarn.user";
+    public final static String YARN_HDFS= "yarn.hdfs";
+
+	public boolean isValidMaster() {
+		return containsKey(HELIX_ZOOKEEPER) &&
+			   containsKey(HELIX_CLUSTER) &&
+			   containsKey(PROVIDER_METADATA) &&
+			   containsKey(PROVIDER_NAME) &&
+			   containsKey(YARN_RESOURCEMANAGER) &&
+			   containsKey(YARN_SCHEDULER) &&
+			   containsKey(YARN_USER) &&
+			   containsKey(YARN_HDFS);
+	}
+	
+	public boolean isValidContainer() {
+		return containsKey(HELIX_ZOOKEEPER) &&
+			   containsKey(HELIX_CLUSTER) &&
+			   containsKey(PROVIDER_METADATA) &&
+			   containsKey(CONTAINER_ID);
+	}
+	
+	public String getHelixZookeeper() {
+		return getProperty(HELIX_ZOOKEEPER);
+	}
+
+	public String getHelixCluster() {
+		return getProperty(HELIX_CLUSTER);
+	}
+
+	public String getProviderMetadata() {
+		return getProperty(PROVIDER_METADATA);
+	}
+
+	public String getProviderName() {
+		return getProperty(PROVIDER_NAME);
+	}
+	
+	public String getContainerId() {
+		return getProperty(CONTAINER_ID);
+	}
+	
+    public String getYarnResourceManager() {
+        return getProperty(YARN_RESOURCEMANAGER);
+    }
+
+    public String getYarnScheduler() {
+        return getProperty(YARN_SCHEDULER);
+    }
+
+    public String getYarnUser() {
+        return getProperty(YARN_USER);
+    }
+
+    public String getYarnHdfs() {
+        return getProperty(YARN_HDFS);
+    }
+
+    @Override
+    public Object get(Object key) {
+        Preconditions.checkState(containsKey(key));
+        return super.get(key);
+    }
+    
+    @Override
+    public String getProperty(String key) {
+        Preconditions.checkState(containsKey(key));
+        return super.getProperty(key);
+    }
+    
+}

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerData.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerData.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerData.java
new file mode 100644
index 0000000..d369a2d
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerData.java
@@ -0,0 +1,86 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+/**
+ * Container meta data for YARN-based containers. Reflect lifecycle of container
+ * from requesting, to bootstrapping, active operation and shutdown. Read and
+ * written by {@link YarnMasterProcess}, {@link YarnContainerProvider} and
+ * {@link YarnContainerService}. Also read by {@link YarnStatusProvider}.
+ * Typically stored in zookeeper
+ * 
+ */
+class YarnContainerData {
+
+	static enum ContainerState {
+		ACQUIRE,
+		CONNECTING,
+		ACTIVE,
+		TEARDOWN,
+		FAILED,
+		HALTED,
+		FINALIZE
+	}
+	
+    String                         id;
+    ContainerState                 state;
+    int                            yarnId;
+    String                         owner;
+    YarnContainerProcessProperties properties;
+
+    public YarnContainerData() {
+        // left blank
+    }
+
+    public YarnContainerData(String id, String owner, YarnContainerProcessProperties properties) {
+        this.id = id;
+        this.state = ContainerState.ACQUIRE;
+        this.yarnId = -1;
+        this.owner = owner;
+        this.properties = properties;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public YarnContainerData setId(String id) {
+        this.id = id;
+        return this;
+    }
+
+    public ContainerState getState() {
+        return state;
+    }
+
+    public YarnContainerData setState(ContainerState state) {
+        this.state = state;
+        return this;
+    }
+
+    public int getYarnId() {
+        return yarnId;
+    }
+
+    public YarnContainerData setYarnId(int yarnId) {
+        this.yarnId = yarnId;
+        return this;
+    }
+
+    public String getOwner() {
+        return owner;
+    }
+
+    public YarnContainerData setOwner(String owner) {
+        this.owner = owner;
+        return this;
+    }
+
+    public YarnContainerProcessProperties getProperties() {
+        return properties;
+    }
+
+    public YarnContainerData setProperties(YarnContainerProcessProperties properties) {
+        this.properties = properties;
+        return this;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProcess.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProcess.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProcess.java
new file mode 100644
index 0000000..2cad52d
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProcess.java
@@ -0,0 +1,53 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+import org.apache.helix.metamanager.container.ContainerProcess;
+import org.apache.log4j.Logger;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Host process for {@link ContainerProcess}es spawned by
+ * {@link YarnContainerProvider}. Configured via *.properties file in working
+ * directory. Corresponds to regular container in YARN and is managed jointly by
+ * the application master and the Helix participant. (Program entry point)
+ * 
+ */
+class YarnContainerProcess {
+    static final Logger log = Logger.getLogger(YarnContainerProcess.class);
+
+    public static void main(String[] args) throws Exception {
+        log.trace("BEGIN YarnProcess.main()");
+
+        final YarnContainerProcessProperties properties = YarnUtils.createContainerProcessProperties(YarnUtils
+                .getPropertiesFromPath(YarnUtils.YARN_CONTAINER_PROPERTIES));
+        Preconditions.checkArgument(properties.isValid(), "container properties not valid: %s", properties.toString());
+
+        log.debug("Launching yarndata service");
+        final ZookeeperYarnDataProvider metaService = new ZookeeperYarnDataProvider(properties.getYarnData());
+        metaService.start();
+
+        log.debug("Launching yarn container service");
+        final YarnContainerService yarnService = new YarnContainerService();
+        yarnService.configure(properties);
+        yarnService.setYarnDataProvider(metaService);
+        yarnService.start();
+
+        log.debug("Installing shutdown hooks");
+        Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+            @Override
+            public void run() {
+                log.debug("Running shutdown hook");
+                yarnService.stop();
+                metaService.stop();
+            }
+        }));
+
+        System.out.println("Press ENTER to stop container process");
+        System.in.read();
+
+        log.debug("Stopping container services");
+        System.exit(0);
+
+        log.trace("END YarnProcess.main()");
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProcessProperties.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProcessProperties.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProcessProperties.java
new file mode 100644
index 0000000..5277e2f
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProcessProperties.java
@@ -0,0 +1,40 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+import org.apache.helix.metamanager.container.ContainerProcessProperties;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Base configuration for {@link YarnContainerProcess}. 
+ *
+ */
+public class YarnContainerProcessProperties extends ContainerProcessProperties {
+	/**
+	 * 
+	 */
+	private static final long serialVersionUID = -2209509977839674160L;
+	
+	public final static String YARNDATA = "yarndata";
+	
+	public boolean isValid() {
+		return super.isValid() &&
+		       containsKey(YARNDATA);
+	}
+	
+	public String getYarnData() {
+		return getProperty(YARNDATA);
+	}
+
+    @Override
+    public Object get(Object key) {
+        Preconditions.checkState(containsKey(key));
+        return super.get(key);
+    }
+    
+    @Override
+    public String getProperty(String key) {
+        Preconditions.checkState(containsKey(key));
+        return super.getProperty(key);
+    }
+    
+}

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProvider.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProvider.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProvider.java
new file mode 100644
index 0000000..c229a26
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProvider.java
@@ -0,0 +1,143 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.helix.metamanager.ContainerProvider;
+import org.apache.helix.metamanager.ContainerProviderService;
+import org.apache.helix.metamanager.impl.yarn.YarnContainerData.ContainerState;
+import org.apache.log4j.Logger;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * {@link ContainerProvider} spawning YARN-based containers. Reads and writes
+ * meta data using {@link YarnDataProvider}. Works in a distributed setting, but
+ * typically requires access to zookeeper.
+ * 
+ */
+class YarnContainerProvider implements ContainerProviderService {
+	
+	static final Logger log = Logger.getLogger(YarnContainerProvider.class);
+
+	static final long POLL_INTERVAL = 1000;
+	static final long CONTAINER_TIMEOUT = 60000;
+	
+	/*
+	 * CONTAINERS
+	 *   A (A, READY)
+	 *   B (B, RUNNING)
+	 */
+	
+	final Object notifier = new Object();
+    final Map<String, Properties> types = new HashMap<String, Properties>();
+	
+	ZookeeperYarnDataProvider yarnDataService;
+	YarnContainerProviderProcess yarnApp;
+	YarnContainerProviderProperties properties;
+	
+    @Override
+    public void configure(Properties properties) throws Exception {
+        YarnContainerProviderProperties yarnProps = new YarnContainerProviderProperties();
+        yarnProps.putAll(properties);
+        configure(yarnProps);
+    }
+    
+    private void configure(YarnContainerProviderProperties properties) {
+        this.properties = properties;
+        
+        for(String containerType : properties.getContainers()) {
+            registerType(containerType, properties.getContainer(containerType));
+        }
+    }
+    
+    @Override
+    public void start() throws Exception {
+        Preconditions.checkNotNull(properties);
+        Preconditions.checkState(properties.isValid(), "provider properties not valid: %s", properties);
+        
+        log.debug("Starting yarn container provider service");
+        yarnDataService = new ZookeeperYarnDataProvider();
+        yarnDataService.configure(properties);
+        yarnDataService.start();
+    }
+    
+    @Override
+    public void stop() throws Exception {
+        log.debug("Stopping yarn container provider service");
+        destroyAll();
+        
+        if(yarnDataService != null) {
+            yarnDataService.stop();
+            yarnDataService = null;
+        }
+    }
+    
+	@Override
+	public void create(final String id, final String type) throws Exception {
+	    Preconditions.checkArgument(types.containsKey(type), "Container type '%s' is not configured", type);
+	    
+		YarnContainerProcessProperties containerProperties = YarnUtils.createContainerProcessProperties(types.get(type));
+
+        log.info(String.format("Running container '%s' (properties='%s')", id, containerProperties));
+        
+		yarnDataService.create(new YarnContainerData(id, properties.getName(), containerProperties));
+		waitForState(id, ContainerState.ACTIVE);
+	}
+
+	@Override
+	public void destroy(final String id) throws Exception {
+		YarnContainerData meta = yarnDataService.read(id);
+
+		if(meta.state == ContainerState.ACTIVE) {
+			log.info(String.format("Destroying active container, going to teardown"));
+			yarnDataService.update(meta.setState(ContainerState.TEARDOWN));
+			
+		} else if(meta.state == ContainerState.FAILED) {
+			log.info(String.format("Destroying failed container, going to teardown"));
+			yarnDataService.update(meta.setState(ContainerState.TEARDOWN));
+			
+		} else if(meta.state == ContainerState.FINALIZE) {
+			log.info(String.format("Destroying finalized container, skipping"));
+			
+		} else {
+			throw new IllegalStateException(String.format("Container '%s' must be active, failed or finalized", id));
+		}
+		
+		waitForState(id, ContainerState.FINALIZE);
+		yarnDataService.delete(id);
+	}
+
+	@Override
+	public void destroyAll() {
+		try {
+			for(YarnContainerData meta : yarnDataService.readAll()) {
+			    if(meta.owner.equals(properties.getName())) {
+			        try { destroy(meta.id); } catch (Exception ignore) {}
+			    }
+			}
+		} catch (Exception ignore) {
+			// ignore
+		}
+	}
+
+	void waitForState(String id, ContainerState state) throws Exception, InterruptedException, TimeoutException {
+		long limit = System.currentTimeMillis() + CONTAINER_TIMEOUT;
+		YarnContainerData meta = yarnDataService.read(id);
+		while(meta.state != state) {
+			if(System.currentTimeMillis() >= limit) {
+				throw new TimeoutException(String.format("Container '%s' failed to reach state '%s' (currently is '%s')", id, state, meta.state));
+			}
+			Thread.sleep(POLL_INTERVAL);
+			meta = yarnDataService.read(id);
+		}
+	}
+	
+    void registerType(String name, Properties properties) {
+        log.debug(String.format("Registering container type '%s' (properties='%s')", name, properties));
+        types.put(name, properties);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProviderProcess.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProviderProcess.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProviderProcess.java
new file mode 100644
index 0000000..d883dce
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProviderProcess.java
@@ -0,0 +1,158 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+import java.io.File;
+import java.net.InetSocketAddress;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.ClientRMProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.helix.metamanager.Service;
+import org.apache.log4j.Logger;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Configurable and runnable service for {@link YarnContainerProvider}
+ * 
+ */
+public class YarnContainerProviderProcess implements Service {
+
+    static final Logger             log                 = Logger.getLogger(YarnContainerProviderProcess.class);
+
+    static String                   YARN_MASTER_COMMAND = "/bin/sh %s 1>%s/stdout 2>%s/stderr";
+
+    Configuration                   conf;
+    YarnRPC                         rpc;
+    ClientRMProtocol                rmClient;
+    ApplicationId                   appId;
+    File                            propertiesFile;
+
+    YarnContainerProviderProperties properties;
+
+    @Override
+    public void configure(Properties properties) throws Exception {
+        configure(YarnUtils.createContainerProviderProperties(properties));
+    }
+
+    private void configure(YarnContainerProviderProperties properties) {
+        this.conf = new YarnConfiguration();
+        this.conf.set(YarnConfiguration.RM_ADDRESS, properties.getResourceManager());
+        this.conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, properties.getScheduler());
+        this.conf.set(FileSystem.FS_DEFAULT_NAME_KEY, properties.getHdfs());
+
+        this.rpc = YarnRPC.create(conf);
+
+        this.properties = properties;
+    }
+
+    @Override
+    public void start() throws Exception {
+        Preconditions.checkNotNull(properties);
+        Preconditions.checkState(properties.isValid());
+
+        connect();
+
+        String command = String.format(YARN_MASTER_COMMAND, YarnUtils.YARN_MASTER_PATH, ApplicationConstants.LOG_DIR_EXPANSION_VAR,
+                ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+
+        log.info(String.format("Starting application '%s' provider '%s' (masterCommand='%s')", properties.getYarnData(), properties.getName(), command));
+
+        log.debug(String.format("Running master command \"%s\"", command));
+
+        // app id
+        GetNewApplicationRequest appRequest = Records.newRecord(GetNewApplicationRequest.class);
+        GetNewApplicationResponse appResponse = rmClient.getNewApplication(appRequest);
+
+        this.appId = appResponse.getApplicationId();
+
+        log.info(String.format("Acquired app id '%s' for '%s'", appId.toString(), properties.getName()));
+
+        // command
+        ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class);
+        launchContext.setCommands(Collections.singletonList(command));
+
+        // resource limit
+        Resource resource = Records.newRecord(Resource.class);
+        resource.setMemory(256); // TODO make dynamic
+        launchContext.setResource(resource);
+
+        // environment
+        Map<String, String> env = new HashMap<String, String>();
+        launchContext.setEnvironment(env);
+
+        // configuration
+        propertiesFile = YarnUtils.writePropertiesToTemp(properties);
+
+        // HDFS
+        final String namespace = appId.toString();
+        final Path masterArchive = YarnUtils.copyToHdfs(YarnUtils.YARN_MASTER_ARCHIVE_PATH, YarnUtils.YARN_MASTER_STAGING, namespace, conf);
+        final Path masterProperties = YarnUtils.copyToHdfs(propertiesFile.getCanonicalPath(), YarnUtils.YARN_MASTER_PROPERTIES, namespace, conf);
+        final Path containerArchive = YarnUtils.copyToHdfs(YarnUtils.YARN_CONTAINER_ARCHIVE_PATH, YarnUtils.YARN_CONTAINER_STAGING, namespace, conf);
+
+        // local resources
+        Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
+        localResources.put(YarnUtils.YARN_MASTER_DESTINATION, YarnUtils.createHdfsResource(masterArchive, LocalResourceType.ARCHIVE, conf));
+        localResources.put(YarnUtils.YARN_MASTER_PROPERTIES, YarnUtils.createHdfsResource(masterProperties, LocalResourceType.FILE, conf));
+        localResources.put(YarnUtils.YARN_CONTAINER_STAGING, YarnUtils.createHdfsResource(containerArchive, LocalResourceType.FILE, conf));
+
+        launchContext.setLocalResources(localResources);
+
+        // user
+        launchContext.setUser(properties.getUser());
+
+        // app submission
+        ApplicationSubmissionContext subContext = Records.newRecord(ApplicationSubmissionContext.class);
+        subContext.setApplicationId(appId);
+        subContext.setApplicationName(properties.getName());
+        subContext.setAMContainerSpec(launchContext);
+
+        SubmitApplicationRequest subRequest = Records.newRecord(SubmitApplicationRequest.class);
+        subRequest.setApplicationSubmissionContext(subContext);
+
+        log.info(String.format("Starting app id '%s'", appId.toString()));
+
+        rmClient.submitApplication(subRequest);
+
+    }
+
+    @Override
+    public void stop() throws YarnRemoteException {
+        log.info(String.format("Stopping app id '%s'", appId.toString()));
+        KillApplicationRequest killRequest = Records.newRecord(KillApplicationRequest.class);
+        killRequest.setApplicationId(appId);
+
+        rmClient.forceKillApplication(killRequest);
+
+		try { YarnUtils.destroyHdfsNamespace(appId.toString(), conf); } catch(Exception ignore) {}
+		
+        propertiesFile.delete();
+    }
+
+    void connect() {
+        YarnConfiguration yarnConf = new YarnConfiguration(conf);
+        InetSocketAddress rmAddress = NetUtils.createSocketAddr(yarnConf.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS));
+        log.info("Connecting to ResourceManager at: " + rmAddress);
+        this.rmClient = ((ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class, rmAddress, conf));
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProviderProperties.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProviderProperties.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProviderProperties.java
new file mode 100644
index 0000000..95ad0aa
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProviderProperties.java
@@ -0,0 +1,64 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+import org.apache.helix.metamanager.provider.ProviderProperties;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Base configuration for {@link YarnContainerProviderProcess} 
+ *
+ */
+public class YarnContainerProviderProperties extends ProviderProperties {
+	/**
+     * 
+     */
+    private static final long serialVersionUID = -8853614843205587170L;
+    
+	public final static String YARNDATA = "yarndata";
+    public final static String RESOURCEMANAGER = "resourcemananger";
+    public final static String SCHEDULER = "scheduler";
+    public final static String USER = "user";
+    public final static String HDFS = "hdfs";
+    
+	public boolean isValid() {
+		return super.isValid() &&
+		       containsKey(YARNDATA) &&
+			   containsKey(RESOURCEMANAGER) &&
+			   containsKey(SCHEDULER) &&
+			   containsKey(USER) &&
+			   containsKey(HDFS);
+	}
+	
+	public String getYarnData() {
+		return getProperty(YARNDATA);
+	}
+
+    public String getResourceManager() {
+        return getProperty(RESOURCEMANAGER);
+    }
+
+    public String getScheduler() {
+        return getProperty(SCHEDULER);
+    }
+
+    public String getUser() {
+        return getProperty(USER);
+    }
+
+    public String getHdfs() {
+        return getProperty(HDFS);
+    }
+    
+    @Override
+    public String getProperty(String key) {
+        Preconditions.checkState(containsKey(key));
+        return super.getProperty(key);
+    }
+    
+    @Override
+    public Object get(Object key) {
+        Preconditions.checkState(containsKey(key));
+        return super.get(key);
+    }
+    
+}

http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerService.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerService.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerService.java
new file mode 100644
index 0000000..804d6ed
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerService.java
@@ -0,0 +1,156 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+import java.io.File;
+import java.util.Properties;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.helix.metamanager.Service;
+import org.apache.helix.metamanager.container.ContainerProcess;
+import org.apache.helix.metamanager.container.ContainerProcessProperties;
+import org.apache.helix.metamanager.container.ContainerUtils;
+import org.apache.helix.metamanager.impl.yarn.YarnContainerData.ContainerState;
+import org.apache.log4j.Logger;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Configurable and runnable service for YARN-based containers. Continuously
+ * checks container meta data and process state and triggers state changes and
+ * container setup and shutdown.
+ * 
+ */
+class YarnContainerService implements Service {
+    static final Logger            log                       = Logger.getLogger(YarnContainerService.class);
+
+    static final long              CONTAINERSERVICE_INTERVAL = 1000;
+
+    YarnContainerProcessProperties properties;
+
+    YarnDataProvider               metaService;
+    ScheduledExecutorService       executor;
+
+    ContainerProcess               process;
+
+    @Override
+    public void configure(Properties properties) throws Exception {
+        Preconditions.checkNotNull(properties);
+        YarnContainerProcessProperties containerProperties = new YarnContainerProcessProperties();
+        containerProperties.putAll(properties);
+        Preconditions.checkArgument(containerProperties.isValid());
+
+        this.properties = containerProperties;
+    }
+
+    public void setYarnDataProvider(YarnDataProvider metaService) {
+        this.metaService = metaService;
+    }
+
+    @Override
+    public void start() {
+        Preconditions.checkNotNull(metaService);
+        Preconditions.checkNotNull(properties);
+        Preconditions.checkState(properties.isValid());
+
+        log.debug("starting yarn container service");
+
+        executor = Executors.newSingleThreadScheduledExecutor();
+        executor.scheduleAtFixedRate(new ContainerStatusService(), 0, CONTAINERSERVICE_INTERVAL, TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    public void stop() {
+        log.debug("stopping yarn container service");
+
+        if (executor != null) {
+            executor.shutdown();
+            while (!executor.isTerminated()) {
+                try {
+                    Thread.sleep(100);
+                } catch (InterruptedException e) {
+                    // ignore
+                }
+            }
+            executor = null;
+        }
+
+        destroyLocalContainerNamespace();
+    }
+
+    class ContainerStatusService implements Runnable {
+        @Override
+        public void run() {
+            log.info("updating container status");
+
+            try {
+                if (!metaService.exists(properties.getName())) {
+                    log.warn(String.format("YarnData for '%s' does not exist. Terminating yarn service.", properties.getName()));
+                    process.stop();
+                    stop();
+                }
+
+                YarnContainerData meta = metaService.read(properties.getName());
+
+                if (meta.state == ContainerState.CONNECTING) {
+                    log.trace("container connecting");
+                    try {
+                        ContainerProcessProperties containerProperties = meta.getProperties();
+
+                        containerProperties.setProperty(ContainerProcessProperties.CLUSTER, properties.getCluster());
+                        containerProperties.setProperty(ContainerProcessProperties.ADDRESS, properties.getAddress());
+                        containerProperties.setProperty(ContainerProcessProperties.NAME, properties.getName());
+
+                        process = ContainerUtils.createProcess(containerProperties);
+                        process.start();
+                    } catch (Exception e) {
+                        log.error("Failed to start participant, going to failed", e);
+                    }
+
+                    if (process.isActive()) {
+                        log.trace("process active, activating container");
+                        metaService.update(meta.setState(ContainerState.ACTIVE));
+
+                    } else if (process.isFailed()) {
+                        log.trace("process failed, failing container");
+                        metaService.update(meta.setState(ContainerState.FAILED));
+
+                    } else {
+                        log.trace("process state unknown, failing container");
+                        metaService.update(meta.setState(ContainerState.FAILED));
+                    }
+                }
+
+                if (meta.state == ContainerState.ACTIVE) {
+                    log.trace("container active");
+                    if (process.isFailed()) {
+                        log.trace("process failed, failing container");
+                        metaService.update(meta.setState(ContainerState.FAILED));
+
+                    } else if (!process.isActive()) {
+                        log.trace("process not active, halting container");
+                        process.stop();
+                        metaService.update(meta.setState(ContainerState.HALTED));
+                    }
+                }
+
+                if (meta.state == ContainerState.TEARDOWN) {
+                    log.trace("container teardown");
+                    process.stop();
+                    metaService.update(meta.setState(ContainerState.HALTED));
+                }
+
+            } catch (Exception e) {
+                log.error(String.format("Error while updating container '%s' status", properties.getName()), e);
+            }
+        }
+    }
+
+    public static void destroyLocalContainerNamespace() {
+        log.info("cleaning up container directory");
+        FileUtils.deleteQuietly(new File(YarnUtils.YARN_CONTAINER_DESTINATION));
+        FileUtils.deleteQuietly(new File(YarnUtils.YARN_CONTAINER_PROPERTIES));
+    }
+
+}