You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by ki...@apache.org on 2013/09/20 20:30:15 UTC
[06/15] Adding Helix-task-framework and Yarn integration modules
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProcess.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProcess.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProcess.java
new file mode 100644
index 0000000..110fe68
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProcess.java
@@ -0,0 +1,64 @@
+package org.apache.helix.metamanager.impl.local;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.helix.HelixManager;
+import org.apache.helix.HelixManagerFactory;
+import org.apache.helix.InstanceType;
+import org.apache.helix.metamanager.container.ContainerStateModelFactory;
+import org.apache.log4j.Logger;
+
+public class LocalContainerProcess
+{
+ static final Logger log = Logger.getLogger(LocalContainerProcess.class);
+
+ private String clusterName;
+ private String zkAddress;
+ private String instanceName;
+ private HelixManager participantManager;
+
+ public LocalContainerProcess(String clusterName, String zkAddress, String instanceName)
+ {
+ this.clusterName = clusterName;
+ this.zkAddress = zkAddress;
+ this.instanceName = instanceName;
+
+ }
+
+ public void start() throws Exception
+ {
+ log.info("STARTING "+ instanceName);
+ participantManager = HelixManagerFactory.getZKHelixManager(clusterName,
+ instanceName, InstanceType.PARTICIPANT, zkAddress);
+ participantManager.getStateMachineEngine().registerStateModelFactory(
+ "MasterSlave", new ContainerStateModelFactory());
+ participantManager.connect();
+ log.info("STARTED "+ instanceName);
+
+ }
+
+ public void stop()
+ {
+ if (participantManager != null)
+ {
+ participantManager.disconnect();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProvider.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProvider.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProvider.java
new file mode 100644
index 0000000..49c9d42
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProvider.java
@@ -0,0 +1,119 @@
+package org.apache.helix.metamanager.impl.local;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.helix.metamanager.ContainerProvider;
+import org.apache.helix.metamanager.ContainerProviderService;
+import org.apache.helix.metamanager.container.ContainerProcess;
+import org.apache.helix.metamanager.container.ContainerProcessProperties;
+import org.apache.helix.metamanager.container.ContainerUtils;
+import org.apache.helix.metamanager.impl.local.LocalContainerSingleton.LocalProcess;
+import org.apache.helix.metamanager.provider.ProviderProperties;
+import org.apache.log4j.Logger;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * {@link ContainerProvider} spawning VM-local containers. Only works in single-VM
+ * deployments as container metadata is managed via singleton.
+ *
+ * @see LocalContainerSingleton
+ */
+class LocalContainerProvider implements ContainerProviderService {
+
+ static final Logger log = Logger.getLogger(LocalContainerProvider.class);
+
+ final Map<String, Properties> types = new HashMap<String, Properties>();
+
+ String address;
+ String cluster;
+ String name;
+
+ @Override
+ public void configure(Properties properties) throws Exception {
+ ProviderProperties providerProperties = new ProviderProperties();
+ providerProperties.putAll(properties);
+ Preconditions.checkArgument(providerProperties.isValid());
+
+ this.address = providerProperties.getProperty("address");
+ this.cluster = providerProperties.getProperty("cluster");
+ this.name = providerProperties.getProperty("name");
+
+ for (String containerType : providerProperties.getContainers()) {
+ registerType(containerType, providerProperties.getContainer(containerType));
+ }
+ }
+
+ @Override
+ public void start() throws Exception {
+ // left blank
+ }
+
+ @Override
+ public void stop() throws Exception {
+ destroyAll();
+ }
+
+ @Override
+ public void create(String id, String type) throws Exception {
+ Map<String, LocalProcess> processes = LocalContainerSingleton.getProcesses();
+
+ synchronized (processes) {
+ Preconditions.checkState(!processes.containsKey(id), "Process '%s' already exists", id);
+ Preconditions.checkState(types.containsKey(type), "Type '%s' is not registered", type);
+
+ ContainerProcessProperties properties = new ContainerProcessProperties(types.get(type));
+
+ properties.setProperty(ContainerProcessProperties.CLUSTER, cluster);
+ properties.setProperty(ContainerProcessProperties.NAME, id);
+ properties.setProperty(ContainerProcessProperties.ADDRESS, address);
+
+ log.info(String.format("Running container '%s' (properties='%s')", id, properties));
+
+ ContainerProcess process = ContainerUtils.createProcess(properties);
+ process.start();
+
+ processes.put(id, new LocalProcess(id, name, process));
+
+ }
+ }
+
+ @Override
+ public void destroy(String id) throws Exception {
+ Map<String, LocalProcess> processes = LocalContainerSingleton.getProcesses();
+
+ synchronized (processes) {
+ if (!processes.containsKey(id))
+ throw new IllegalArgumentException(String.format("Process '%s' does not exists", id));
+
+ log.info(String.format("Destroying container '%s'", id));
+
+ LocalProcess local = processes.remove(id);
+
+ local.process.stop();
+ }
+ }
+
+ @Override
+ public void destroyAll() {
+ Map<String, LocalProcess> processes = LocalContainerSingleton.getProcesses();
+
+ synchronized (processes) {
+ log.info("Destroying all processes");
+ for (LocalProcess local : new HashSet<LocalProcess>(processes.values())) {
+ if (local.owner.equals(name)) {
+ try { destroy(local.id); } catch (Exception ignore) {}
+ }
+ }
+ }
+ }
+
+ void registerType(String name, Properties properties) {
+ log.debug(String.format("Registering container type '%s' (properties='%s')", name, properties));
+ types.put(name, properties);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProviderProcess.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProviderProcess.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProviderProcess.java
new file mode 100644
index 0000000..ed090bc
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerProviderProcess.java
@@ -0,0 +1,45 @@
+package org.apache.helix.metamanager.impl.local;
+
+import java.util.Properties;
+
+import org.apache.helix.metamanager.Service;
+import org.apache.helix.metamanager.provider.ProviderProcess;
+import org.apache.helix.metamanager.provider.ProviderProperties;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Configurable and runnable service for {@link LocalContainerProvider}.
+ *
+ */
+public class LocalContainerProviderProcess implements Service {
+ LocalContainerProvider provider;
+ ProviderProcess process;
+
+ @Override
+ public void configure(Properties properties) throws Exception {
+ ProviderProperties providerProperties = new ProviderProperties();
+ providerProperties.putAll(properties);
+
+ Preconditions.checkArgument(providerProperties.isValid(), "provider properties not valid (properties='%s')", properties);
+
+ provider = new LocalContainerProvider();
+ provider.configure(properties);
+
+ process = new ProviderProcess();
+ process.configure(providerProperties);
+ process.setConteinerProvider(provider);
+ }
+
+ @Override
+ public void start() throws Exception {
+ provider.start();
+ process.start();
+ }
+
+ @Override
+ public void stop() throws Exception {
+ process.stop();
+ provider.stop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerSingleton.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerSingleton.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerSingleton.java
new file mode 100644
index 0000000..b91a848
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerSingleton.java
@@ -0,0 +1,56 @@
+package org.apache.helix.metamanager.impl.local;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.helix.metamanager.container.ContainerProcess;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Singleton tracking metadata for VM-local containers spawned via
+ * {@link LocalContainerProvider}.
+ *
+ */
+public class LocalContainerSingleton {
+ final static Map<String, LocalProcess> processes = new HashMap<String, LocalProcess>();
+
+ private LocalContainerSingleton() {
+ // left blank
+ }
+
+ public static Map<String, LocalProcess> getProcesses() {
+ return processes;
+ }
+
+ public static void reset() {
+ synchronized (processes) {
+ for (LocalProcess local : processes.values()) {
+ local.process.stop();
+ }
+ processes.clear();
+ }
+ }
+
+ public static void killProcess(String id) throws InterruptedException {
+ synchronized (processes) {
+ Preconditions.checkArgument(processes.containsKey(id), "Process '%s' does not exist", id);
+ ContainerProcess process = processes.get(id).process;
+ process.stop();
+ processes.remove(id);
+ }
+ }
+
+ static class LocalProcess {
+ final String id;
+ final String owner;
+ final ContainerProcess process;
+
+ public LocalProcess(String id, String owner, ContainerProcess process) {
+ this.id = id;
+ this.owner = owner;
+ this.process = process;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerStatusProvider.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerStatusProvider.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerStatusProvider.java
new file mode 100644
index 0000000..604ad25
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalContainerStatusProvider.java
@@ -0,0 +1,37 @@
+package org.apache.helix.metamanager.impl.local;
+
+import java.util.Map;
+
+import org.apache.helix.metamanager.ContainerStatusProvider;
+import org.apache.helix.metamanager.impl.local.LocalContainerSingleton.LocalProcess;
+
+public class LocalContainerStatusProvider implements ContainerStatusProvider {
+
+ @Override
+ public boolean exists(String id) {
+ Map<String, LocalProcess> processes = LocalContainerSingleton.getProcesses();
+
+ synchronized (processes) {
+ return processes.containsKey(id);
+ }
+ }
+
+ @Override
+ public boolean isActive(String id) {
+ Map<String, LocalProcess> processes = LocalContainerSingleton.getProcesses();
+
+ synchronized (processes) {
+ return processes.get(id).process.isActive();
+ }
+ }
+
+ @Override
+ public boolean isFailed(String id) {
+ Map<String, LocalProcess> processes = LocalContainerSingleton.getProcesses();
+
+ synchronized (processes) {
+ return processes.get(id).process.isFailed();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalStatusProvider.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalStatusProvider.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalStatusProvider.java
new file mode 100644
index 0000000..c53a3ba
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/local/LocalStatusProvider.java
@@ -0,0 +1,53 @@
+package org.apache.helix.metamanager.impl.local;
+
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.helix.metamanager.StatusProviderService;
+import org.apache.helix.metamanager.impl.local.LocalContainerSingleton.LocalProcess;
+
+/**
+ * StatusProvider for VM-local containers spawned via
+ * {@link LocalContainerProvider}. Runnable and configurable service.
+ *
+ */
+public class LocalStatusProvider implements StatusProviderService {
+
+ @Override
+ public boolean exists(String id) {
+ Map<String, LocalProcess> processes = LocalContainerSingleton.getProcesses();
+
+ synchronized (processes) {
+ return processes.containsKey(id);
+ }
+ }
+
+ @Override
+ public boolean isHealthy(String id) {
+ Map<String, LocalProcess> processes = LocalContainerSingleton.getProcesses();
+
+ synchronized (processes) {
+ LocalProcess local = processes.get(id);
+
+ if (local == null)
+ return false;
+
+ return local.process.isActive();
+ }
+ }
+
+ @Override
+ public void configure(Properties properties) throws Exception {
+ // left blank
+ }
+
+ @Override
+ public void start() throws Exception {
+ // left blank
+ }
+
+ @Override
+ public void stop() throws Exception {
+ // left blank
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProcess.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProcess.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProcess.java
new file mode 100644
index 0000000..0069110
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProcess.java
@@ -0,0 +1,93 @@
+package org.apache.helix.metamanager.impl.shell;
+
+import java.io.File;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.helix.metamanager.container.ContainerProcess;
+import org.apache.helix.metamanager.container.ContainerProcessProperties;
+import org.apache.helix.metamanager.container.ContainerUtils;
+import org.apache.log4j.Logger;
+
+/**
+ * Host process for Shell-based container. ContainerProcess configuration is
+ * read from path in first command-line argument. Status is maintained using
+ * temporary marker file. (Program entry point)
+ *
+ */
+class ShellContainerProcess {
+ static final Logger log = Logger.getLogger(ShellContainerProcess.class);
+
+ public static final long MONITOR_INTERVAL = 5000;
+
+ static String markerDir;
+ static ContainerProcess process;
+ static ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
+
+ public static void main(String[] args) throws Exception {
+ final String propertiesPath = args[0];
+ markerDir = args[1];
+
+ ContainerProcessProperties properties = ContainerUtils.getPropertiesFromPath(propertiesPath);
+
+ process = ContainerUtils.createProcess(properties);
+
+ log.debug("Installing shutdown hooks");
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ log.debug("Running shutdown hook");
+ try {
+ ShellContainerProcess.stop();
+ } catch (Exception ignore) {
+ }
+ }
+ });
+
+ log.debug("Launching shell container process");
+ process.start();
+
+ ShellUtils.createMarker(new File(markerDir));
+
+ log.debug("Launching process monitor");
+ executor.scheduleAtFixedRate(new ProcessMonitor(), 0, MONITOR_INTERVAL, TimeUnit.MILLISECONDS);
+ }
+
+ static void stop() throws InterruptedException {
+ log.debug("Shutting down shell process");
+ if (process != null) {
+ process.stop();
+ ShellUtils.destroyMarker(new File(markerDir));
+ }
+ if (executor != null) {
+ executor.shutdownNow();
+ while (!executor.isTerminated()) {
+ Thread.sleep(100);
+ }
+ executor = null;
+ }
+ }
+
+ static class ProcessMonitor implements Runnable {
+ @Override
+ public void run() {
+ if (process.isFailed()) {
+ log.warn("detected process failure");
+ try {
+ ShellContainerProcess.stop();
+ } catch (Exception ignore) {
+ }
+ System.exit(1);
+ }
+ if (!process.isActive()) {
+ log.warn("detected process shutdown");
+ try {
+ ShellContainerProcess.stop();
+ } catch (Exception ignore) {
+ }
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProvider.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProvider.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProvider.java
new file mode 100644
index 0000000..69e2553
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProvider.java
@@ -0,0 +1,151 @@
+package org.apache.helix.metamanager.impl.shell;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.helix.metamanager.ContainerProvider;
+import org.apache.helix.metamanager.ContainerProviderService;
+import org.apache.helix.metamanager.container.ContainerProcessProperties;
+import org.apache.helix.metamanager.impl.shell.ShellContainerSingleton.ShellProcess;
+import org.apache.helix.metamanager.provider.ProviderProperties;
+import org.apache.log4j.Logger;
+
+import com.google.common.base.Preconditions;
+import com.google.common.io.Files;
+
+/**
+ * {@link ContainerProvider} spawning shell-based containers. Only works in single-VM
+ * deployments as container metadata is managed via singleton.
+ *
+ * @see ShellContainerSingleton
+ */
+class ShellContainerProvider implements ContainerProviderService {
+
+ static final Logger log = Logger.getLogger(ShellContainerProvider.class);
+
+ static final String RUN_COMMAND = "/bin/sh";
+
+ static final long POLL_INTERVAL = 1000;
+ static final long CONTAINER_TIMEOUT = 60000;
+
+ // global view of processes required
+ static final Map<String, ShellProcess> processes = new HashMap<String, ShellProcess>();
+
+ final Map<String, Properties> types = new HashMap<String, Properties>();
+
+ String address;
+ String cluster;
+ String name;
+
+ @Override
+ public void configure(Properties properties) throws Exception {
+ Preconditions.checkNotNull(properties);
+ ProviderProperties providerProperties = new ProviderProperties();
+ providerProperties.putAll(properties);
+ Preconditions.checkArgument(providerProperties.isValid());
+
+ this.address = providerProperties.getProperty("address");
+ this.cluster = providerProperties.getProperty("cluster");
+ this.name = providerProperties.getProperty("name");
+
+ for (String containerType : providerProperties.getContainers()) {
+ registerType(containerType, providerProperties.getContainer(containerType));
+ }
+ }
+
+ @Override
+ public void start() throws Exception {
+ // left blank
+ }
+
+ @Override
+ public void stop() throws Exception {
+ destroyAll();
+ }
+
+ @Override
+ public void create(String id, String type) throws Exception {
+ Map<String, ShellProcess> processes = ShellContainerSingleton.getProcesses();
+
+ synchronized (processes) {
+ Preconditions.checkState(!processes.containsKey(id), "Process '%s' already exists", id);
+ Preconditions.checkState(types.containsKey(type), "Type '%s' is not registered", type);
+
+ ContainerProcessProperties properties = new ContainerProcessProperties(types.get(type));
+
+ properties.setProperty(ContainerProcessProperties.CLUSTER, cluster);
+ properties.setProperty(ContainerProcessProperties.NAME, id);
+ properties.setProperty(ContainerProcessProperties.ADDRESS, address);
+
+ File tmpDir = Files.createTempDir();
+ File tmpProperties = new File(tmpDir.getCanonicalPath() + File.separator + ShellUtils.SHELL_CONTAINER_PROPERTIES);
+ File tmpMarker = new File(tmpDir.getCanonicalPath());
+
+ properties.store(new FileWriter(tmpProperties), id);
+
+ log.info(String.format("Running container '%s' (properties='%s')", id, properties));
+
+ log.debug(String.format("Invoking command '%s %s %s %s'", RUN_COMMAND, ShellUtils.SHELL_CONTAINER_PATH, tmpProperties.getCanonicalPath(),
+ tmpMarker.getCanonicalPath()));
+
+ ProcessBuilder builder = new ProcessBuilder();
+ builder.command(RUN_COMMAND, ShellUtils.SHELL_CONTAINER_PATH, tmpProperties.getCanonicalPath(), tmpMarker.getCanonicalPath());
+
+ Process process = builder.start();
+
+ processes.put(id, new ShellProcess(id, name, process, tmpDir));
+
+ long limit = System.currentTimeMillis() + CONTAINER_TIMEOUT;
+ while (!ShellUtils.hasMarker(tmpDir)) {
+ if (System.currentTimeMillis() >= limit) {
+ throw new TimeoutException(String.format("Container '%s' failed to reach active state", id));
+ }
+ Thread.sleep(POLL_INTERVAL);
+ }
+ }
+ }
+
+ @Override
+ public void destroy(String id) throws Exception {
+ Map<String, ShellProcess> processes = ShellContainerSingleton.getProcesses();
+
+ synchronized (processes) {
+ if (!processes.containsKey(id))
+ throw new IllegalArgumentException(String.format("Process '%s' does not exists", id));
+
+ log.info(String.format("Destroying container '%s'", id));
+
+ ShellProcess shell = processes.remove(id);
+ shell.process.destroy();
+ shell.process.waitFor();
+
+ FileUtils.deleteDirectory(shell.tmpDir);
+ }
+ }
+
+ @Override
+ public void destroyAll() {
+ Map<String, ShellProcess> processes = ShellContainerSingleton.getProcesses();
+
+ synchronized (processes) {
+ log.info("Destroying all owned processes");
+ for (ShellProcess process : new HashSet<ShellProcess>(processes.values())) {
+ if (process.owner.equals(name)) {
+ try { destroy(process.id); } catch (Exception ignore) {}
+ }
+ }
+ }
+ }
+
+ void registerType(String name, Properties properties) {
+ log.debug(String.format("Registering container type '%s' (properties='%s')", name, properties));
+ types.put(name, properties);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProviderProcess.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProviderProcess.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProviderProcess.java
new file mode 100644
index 0000000..36f150d
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerProviderProcess.java
@@ -0,0 +1,45 @@
+package org.apache.helix.metamanager.impl.shell;
+
+import java.util.Properties;
+
+import org.apache.helix.metamanager.Service;
+import org.apache.helix.metamanager.provider.ProviderProcess;
+import org.apache.helix.metamanager.provider.ProviderProperties;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Configurable and runnable service for {@link ShellContainerProvider}.
+ *
+ */
+public class ShellContainerProviderProcess implements Service {
+ ShellContainerProvider provider;
+ ProviderProcess process;
+
+ @Override
+ public void configure(Properties properties) throws Exception {
+ ProviderProperties providerProperties = new ProviderProperties();
+ providerProperties.putAll(properties);
+
+ Preconditions.checkArgument(providerProperties.isValid(), "provider properties not valid (properties='%s')", properties);
+
+ provider = new ShellContainerProvider();
+ provider.configure(properties);
+
+ process = new ProviderProcess();
+ process.configure(providerProperties);
+ process.setConteinerProvider(provider);
+ }
+
+ @Override
+ public void start() throws Exception {
+ provider.start();
+ process.start();
+ }
+
+ @Override
+ public void stop() throws Exception {
+ process.stop();
+ provider.stop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerSingleton.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerSingleton.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerSingleton.java
new file mode 100644
index 0000000..91054e1
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerSingleton.java
@@ -0,0 +1,58 @@
+package org.apache.helix.metamanager.impl.shell;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Singleton tracking metadata for shell-based containers spawned via
+ * {@link ShellContainerProvider}.
+ *
+ */
+public class ShellContainerSingleton {
+ static final Map<String, ShellProcess> processes = new HashMap<String, ShellProcess>();
+
+ private ShellContainerSingleton() {
+ // left blank
+ }
+
+ public static Map<String, ShellProcess> getProcesses() {
+ return processes;
+ }
+
+ public static void reset() {
+ synchronized (processes) {
+ for (ShellProcess shell : processes.values()) {
+ shell.process.destroy();
+ try { shell.process.waitFor(); } catch(Exception ignore) {}
+ }
+ processes.clear();
+ }
+ }
+
+ public static void killProcess(String id) throws InterruptedException {
+ synchronized (processes) {
+ Preconditions.checkArgument(processes.containsKey(id), "Process '%s' does not exist", id);
+ Process process = processes.get(id).process;
+ process.destroy();
+ process.waitFor();
+ processes.remove(id);
+ }
+ }
+
+ static class ShellProcess {
+ final String id;
+ final String owner;
+ final Process process;
+ final File tmpDir;
+
+ public ShellProcess(String id, String owner, Process process, File tmpDir) {
+ this.id = id;
+ this.owner = owner;
+ this.process = process;
+ this.tmpDir = tmpDir;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerStatusProvider.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerStatusProvider.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerStatusProvider.java
new file mode 100644
index 0000000..03e55c6
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellContainerStatusProvider.java
@@ -0,0 +1,52 @@
+package org.apache.helix.metamanager.impl.shell;
+
+import java.util.Map;
+
+import org.apache.helix.metamanager.ContainerStatusProvider;
+import org.apache.helix.metamanager.impl.shell.ShellContainerSingleton.ShellProcess;
+
+public class ShellContainerStatusProvider implements ContainerStatusProvider {
+
+ @Override
+ public boolean exists(String id) {
+ Map<String, ShellProcess> processes = ShellContainerSingleton.getProcesses();
+
+ synchronized (processes) {
+ return processes.containsKey(id);
+ }
+ }
+
+ @Override
+ public boolean isActive(String id) {
+ Map<String, ShellProcess> processes = ShellContainerSingleton.getProcesses();
+
+ synchronized (processes) {
+ ShellProcess shell = processes.get(id);
+
+ try {
+ shell.process.exitValue();
+ return false;
+ } catch (IllegalThreadStateException e) {
+ // still running
+ return true;
+ }
+ }
+ }
+
+ @Override
+ public boolean isFailed(String id) {
+ Map<String, ShellProcess> processes = ShellContainerSingleton.getProcesses();
+
+ synchronized (processes) {
+ ShellProcess shell = processes.get(id);
+
+ try {
+ return (shell.process.exitValue() != 0);
+ } catch (IllegalThreadStateException e) {
+ // still running
+ return false;
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellStatusProvider.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellStatusProvider.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellStatusProvider.java
new file mode 100644
index 0000000..015218b
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellStatusProvider.java
@@ -0,0 +1,64 @@
+package org.apache.helix.metamanager.impl.shell;
+
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.helix.metamanager.StatusProviderService;
+import org.apache.helix.metamanager.impl.shell.ShellContainerSingleton.ShellProcess;
+
+/**
+ * StatusProvider for shell-based containers spawned via
+ * {@link ShellContainerProvider}. Runnable and configurable service.
+ *
+ */
+public class ShellStatusProvider implements StatusProviderService {
+
+ @Override
+ public boolean exists(String id) {
+ Map<String, ShellProcess> processes = ShellContainerSingleton.getProcesses();
+
+ synchronized (processes) {
+ return processes.containsKey(id);
+ }
+ }
+
+ @Override
+ public boolean isHealthy(String id) {
+ Map<String, ShellProcess> processes = ShellContainerSingleton.getProcesses();
+
+ synchronized (processes) {
+ ShellProcess shell = processes.get(id);
+
+ if (shell == null)
+ return false;
+
+ if (!ShellUtils.hasMarker(shell.tmpDir))
+ return false;
+
+ try {
+ // exit value
+ shell.process.exitValue();
+ return false;
+ } catch (IllegalThreadStateException e) {
+ // expected
+ }
+
+ return true;
+ }
+ }
+
+ @Override
+ public void configure(Properties properties) throws Exception {
+ // left blank
+ }
+
+ @Override
+ public void start() throws Exception {
+ // left blank
+ }
+
+ @Override
+ public void stop() throws Exception {
+ // left blank
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellUtils.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellUtils.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellUtils.java
new file mode 100644
index 0000000..dcec2ae
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/shell/ShellUtils.java
@@ -0,0 +1,54 @@
+package org.apache.helix.metamanager.impl.shell;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.log4j.Logger;
+
+/**
+ * Utility for creating and destroying temporary marker files for shell-based
+ * containers.
+ *
+ */
+class ShellUtils {
+
+ static final Logger log = Logger.getLogger(ShellUtils.class);
+
+ static final String SHELL_CONTAINER_PATH = "target/metamanager-pkg/bin/shell-container-process.sh";
+ static final String SHELL_CONTAINER_PROPERTIES = "container.properties";
+ static final String SHELL_CONTAINER_MARKER = "active";
+
+ private ShellUtils() {
+ // left blank
+ }
+
+ public static boolean hasMarker(File processDir) {
+ try {
+ log.debug(String.format("checking for marker file '%s'", getMarkerFile(processDir)));
+ if (getMarkerFile(processDir).exists())
+ return true;
+ } catch (IOException e) {
+ // ignore
+ }
+ return false;
+ }
+
+ public static void createMarker(File processDir) throws IOException {
+ log.debug(String.format("creating marker file '%s'", getMarkerFile(processDir)));
+ getMarkerFile(processDir).createNewFile();
+ }
+
+ public static void destroyMarker(File processDir) {
+ try {
+ log.debug(String.format("destroying marker file '%s'", getMarkerFile(processDir)));
+ getMarkerFile(processDir).delete();
+ } catch (IOException e) {
+ // ignore
+ }
+ }
+
+ public static File getMarkerFile(File processDir) throws IOException {
+ return new File(processDir.getCanonicalPath() + File.separatorChar + SHELL_CONTAINER_MARKER);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/ApplicationConfig.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/ApplicationConfig.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/ApplicationConfig.java
new file mode 100644
index 0000000..9c85232
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/ApplicationConfig.java
@@ -0,0 +1,32 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+public class ApplicationConfig {
+ final String clusterAddress;
+ final String clusterName;
+ final String metadataAddress;
+ final String providerName;
+
+ public ApplicationConfig(String clusterAddress, String clusterName,
+ String metadataAddress, String providerName) {
+ this.clusterAddress = clusterAddress;
+ this.clusterName = clusterName;
+ this.metadataAddress = metadataAddress;
+ this.providerName = providerName;
+ }
+
+ public String getClusterAddress() {
+ return clusterAddress;
+ }
+
+ public String getClusterName() {
+ return clusterName;
+ }
+
+ public String getMetadataAddress() {
+ return metadataAddress;
+ }
+
+ public String getProviderName() {
+ return providerName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/ContainerMetadata.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/ContainerMetadata.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/ContainerMetadata.java
new file mode 100644
index 0000000..7b25d31
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/ContainerMetadata.java
@@ -0,0 +1,80 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+
+class ContainerMetadata {
+
+ static enum ContainerState {
+ ACQUIRE,
+ CONNECTING,
+ ACTIVE,
+ TEARDOWN,
+ FAILED,
+ HALTED,
+ FINALIZE
+ }
+
+ String id;
+ ContainerState state;
+ int yarnId;
+ String owner;
+ YarnContainerProcessProperties properties;
+
+ public ContainerMetadata() {
+ // left blank
+ }
+
+ public ContainerMetadata(String id, String owner, YarnContainerProcessProperties properties) {
+ this.id = id;
+ this.state = ContainerState.ACQUIRE;
+ this.yarnId = -1;
+ this.owner = owner;
+ this.properties = properties;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public ContainerMetadata setId(String id) {
+ this.id = id;
+ return this;
+ }
+
+ public ContainerState getState() {
+ return state;
+ }
+
+ public ContainerMetadata setState(ContainerState state) {
+ this.state = state;
+ return this;
+ }
+
+ public int getYarnId() {
+ return yarnId;
+ }
+
+ public ContainerMetadata setYarnId(int yarnId) {
+ this.yarnId = yarnId;
+ return this;
+ }
+
+ public String getOwner() {
+ return owner;
+ }
+
+ public ContainerMetadata setOwner(String owner) {
+ this.owner = owner;
+ return this;
+ }
+
+ public YarnContainerProcessProperties getProperties() {
+ return properties;
+ }
+
+ public ContainerMetadata setProperties(YarnContainerProcessProperties properties) {
+ this.properties = properties;
+ return this;
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/MetadataProvider.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/MetadataProvider.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/MetadataProvider.java
new file mode 100644
index 0000000..a35c16e
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/MetadataProvider.java
@@ -0,0 +1,42 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+import java.util.Collection;
+
+interface MetadataProvider {
+
+ public boolean exists(String id);
+
+ public void create(ContainerMetadata meta) throws MetadataException;
+
+ public ContainerMetadata read(String id) throws MetadataException;
+
+ public Collection<ContainerMetadata> readAll() throws MetadataException;
+
+ public void update(ContainerMetadata meta) throws MetadataException;
+
+ public void delete(String id) throws MetadataException;
+
+ public static class MetadataException extends Exception {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = -2846997013918977056L;
+
+ public MetadataException() {
+ super();
+ }
+
+ public MetadataException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public MetadataException(String message) {
+ super(message);
+ }
+
+ public MetadataException(Throwable cause) {
+ super(cause);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/MetadataService.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/MetadataService.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/MetadataService.java
new file mode 100644
index 0000000..35dd1ee
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/MetadataService.java
@@ -0,0 +1,42 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+import java.util.Collection;
+
+public interface MetadataService {
+
+ public boolean exists(String id);
+
+ public void create(ContainerMetadata meta) throws MetadataServiceException;
+
+ public ContainerMetadata read(String id) throws MetadataServiceException;
+
+ public Collection<ContainerMetadata> readAll() throws MetadataServiceException;
+
+ public void update(ContainerMetadata meta) throws MetadataServiceException;
+
+ public void delete(String id) throws MetadataServiceException;
+
+ public static class MetadataServiceException extends Exception {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = -2846997013918977056L;
+
+ public MetadataServiceException() {
+ super();
+ }
+
+ public MetadataServiceException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public MetadataServiceException(String message) {
+ super(message);
+ }
+
+ public MetadataServiceException(Throwable cause) {
+ super(cause);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/Utils.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/Utils.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/Utils.java
new file mode 100644
index 0000000..99f9a03
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/Utils.java
@@ -0,0 +1,94 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.helix.metamanager.impl.yarn.ContainerMetadata.ContainerState;
+import org.apache.log4j.Logger;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.TypeAdapter;
+import com.google.gson.stream.JsonReader;
+import com.google.gson.stream.JsonToken;
+import com.google.gson.stream.JsonWriter;
+
+public class Utils {
+
+ static final Logger log = Logger.getLogger(Utils.class);
+
+ static Gson gson;
+ static {
+ GsonBuilder builder = new GsonBuilder();
+ builder.registerTypeAdapter(ContainerState.class, new ContainerStateAdapter());
+ builder.setPrettyPrinting();
+ gson = builder.create();
+ }
+
+ static Map<String, LocalResource> dummyResources = createDummyResources();
+
+ static String toJson(ContainerMetadata meta) {
+ return gson.toJson(meta);
+ }
+
+ static ContainerMetadata fromJson(String json) {
+ return gson.fromJson(json, ContainerMetadata.class);
+ }
+
+ static Map<String, LocalResource> getDummyResources() {
+ return dummyResources;
+ }
+
+ private static Map<String, LocalResource> createDummyResources() {
+ File dummy = new File("/tmp/dummy");
+
+ if(!dummy.exists()) {
+ try {
+ dummy.createNewFile();
+ } catch(Exception e) {
+ log.error("could not create dummy file", e);
+ System.exit(1);
+ }
+ }
+
+ Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
+ Path path = new Path(dummy.toURI());
+ LocalResource localResource = Records.newRecord(LocalResource.class);
+ localResource.setType(LocalResourceType.FILE);
+ localResource.setVisibility(LocalResourceVisibility.APPLICATION);
+ localResource.setResource(ConverterUtils.getYarnUrlFromPath(path));
+ localResource.setTimestamp(dummy.lastModified());
+ localResource.setSize(dummy.length());
+ localResources.put("dummy", localResource);
+ return localResources;
+ }
+
+ static class ContainerStateAdapter extends TypeAdapter<ContainerState> {
+ @Override
+ public ContainerState read(JsonReader reader) throws IOException {
+ if (reader.peek() == JsonToken.NULL) {
+ reader.nextNull();
+ return null;
+ }
+ return ContainerState.valueOf(reader.nextString());
+ }
+
+ @Override
+ public void write(JsonWriter writer, ContainerState value) throws IOException {
+ if (value == null) {
+ writer.nullValue();
+ return;
+ }
+ writer.value(value.name());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnApplication.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnApplication.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnApplication.java
new file mode 100644
index 0000000..3b7dcd6
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnApplication.java
@@ -0,0 +1,171 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+import java.io.File;
+import java.net.InetSocketAddress;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.ClientRMProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.helix.metamanager.ConfigTool;
+import org.apache.helix.metamanager.Service;
+import org.apache.log4j.Logger;
+
+class YarnApplication implements Service {
+
+ static final Logger log = Logger.getLogger(YarnApplication.class);
+
+ static final String ENV_CLUSTER_ADDRESS = "YA_CLUSTER_ADDRESS";
+ static final String ENV_CLUSTER_NAME = "YA_CLUSTER_NAME";
+ static final String ENV_METADATA_ADDRESS = "YA_METADATA_ADDRESS";
+ static final String ENV_PROVIDER_NAME = "YA_PROVIDER_NAME";
+
+ static String YARN_MASTER_COMMAND = "/bin/sh %s 1>%s/stdout 2>%s/stderr";
+
+ Configuration conf;
+ YarnRPC rpc;
+ ClientRMProtocol rmClient;
+ ApplicationId appId;
+ File propertiesFile;
+
+ YarnApplicationProperties properties;
+
+ public YarnApplication() {
+ // left blank
+ }
+
+ public YarnApplication(YarnApplicationProperties properties) {
+ this.properties = properties;
+ internalConf();
+ }
+
+ @Override
+ public void configure(Properties properties) throws Exception {
+ YarnApplicationProperties yarnProps = new YarnApplicationProperties();
+ yarnProps.putAll(properties);
+ this.properties = yarnProps;
+ internalConf();
+ }
+
+ public void internalConf() {
+ this.conf = new YarnConfiguration();
+ this.conf.set(YarnConfiguration.RM_ADDRESS, properties.getYarnResourceManager());
+ this.conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, properties.getYarnScheduler());
+ this.conf.set(FileSystem.FS_DEFAULT_NAME_KEY, properties.getYarnHdfs());
+
+ this.rpc = YarnRPC.create(conf);
+ }
+
+ @Override
+ public void start() throws Exception {
+ connect();
+
+ String command = String.format(YARN_MASTER_COMMAND, ConfigTool.YARN_MASTER_PATH,
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+
+ log.info(String.format("Starting application '%s' provider '%s' (masterCommand='%s')",
+ properties.getProviderMetadata(), properties.getProviderName(), command));
+
+ log.debug(String.format("Running master command \"%s\"", command));
+
+ // app id
+ GetNewApplicationRequest appRequest = Records.newRecord(GetNewApplicationRequest.class);
+ GetNewApplicationResponse appResponse = rmClient.getNewApplication(appRequest);
+
+ this.appId = appResponse.getApplicationId();
+
+ log.info(String.format("Acquired app id '%s' for '%s'", appId.toString(), properties.getProviderName()));
+
+ // command
+ ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class);
+ launchContext.setCommands(Collections.singletonList(command));
+
+ // resource limit
+ Resource resource = Records.newRecord(Resource.class);
+ resource.setMemory(256); // TODO make dynamic
+ launchContext.setResource(resource);
+
+ // environment
+ Map<String, String> env = new HashMap<String, String>();
+ launchContext.setEnvironment(env);
+
+ // configuration
+ propertiesFile = YarnUtils.writePropertiesToTemp(properties);
+
+ // HDFS
+ final String namespace = appId.toString();
+ final Path masterArchive = YarnUtils.copyToHdfs(ConfigTool.YARN_MASTER_ARCHIVE_PATH, ConfigTool.YARN_MASTER_STAGING, namespace, conf);
+ final Path masterProperties = YarnUtils.copyToHdfs(propertiesFile.getCanonicalPath(), ConfigTool.YARN_MASTER_PROPERTIES, namespace, conf);
+ final Path containerArchive = YarnUtils.copyToHdfs(ConfigTool.YARN_CONTAINER_ARCHIVE_PATH, ConfigTool.YARN_CONTAINER_STAGING, namespace, conf);
+
+ // local resources
+ Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
+ localResources.put(ConfigTool.YARN_MASTER_DESTINATION,
+ YarnUtils.createHdfsResource(masterArchive, LocalResourceType.ARCHIVE, conf));
+ localResources.put(ConfigTool.YARN_MASTER_PROPERTIES,
+ YarnUtils.createHdfsResource(masterProperties, LocalResourceType.FILE, conf));
+ localResources.put(ConfigTool.YARN_CONTAINER_STAGING,
+ YarnUtils.createHdfsResource(containerArchive, LocalResourceType.FILE, conf));
+
+ launchContext.setLocalResources(localResources);
+
+ // user
+ launchContext.setUser(properties.getYarnUser());
+
+ // app submission
+ ApplicationSubmissionContext subContext = Records.newRecord(ApplicationSubmissionContext.class);
+ subContext.setApplicationId(appId);
+ subContext.setApplicationName(properties.getProviderName());
+ subContext.setAMContainerSpec(launchContext);
+
+ SubmitApplicationRequest subRequest = Records.newRecord(SubmitApplicationRequest.class);
+ subRequest.setApplicationSubmissionContext(subContext);
+
+ log.info(String.format("Starting app id '%s'", appId.toString()));
+
+ rmClient.submitApplication(subRequest);
+
+ }
+
+ @Override
+ public void stop() throws YarnRemoteException {
+ log.info(String.format("Stopping app id '%s'", appId.toString()));
+ KillApplicationRequest killRequest = Records.newRecord(KillApplicationRequest.class);
+ killRequest.setApplicationId(appId);
+
+ rmClient.forceKillApplication(killRequest);
+
+ try { YarnUtils.destroyHdfsNamespace(appId.toString(), conf); } catch(Exception ignore) {}
+
+ propertiesFile.delete();
+ }
+
+ void connect() {
+ YarnConfiguration yarnConf = new YarnConfiguration(conf);
+ InetSocketAddress rmAddress = NetUtils.createSocketAddr(yarnConf.get(
+ YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADDRESS));
+ log.info("Connecting to ResourceManager at: " + rmAddress);
+ this.rmClient = ((ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class, rmAddress, conf));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnApplicationProperties.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnApplicationProperties.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnApplicationProperties.java
new file mode 100644
index 0000000..e047179
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnApplicationProperties.java
@@ -0,0 +1,91 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+import java.util.Properties;
+
+import org.apache.helix.metamanager.container.ContainerProcessProperties;
+
+import com.google.common.base.Preconditions;
+
+public class YarnApplicationProperties extends Properties {
+ /**
+ *
+ */
+ private static final long serialVersionUID = -2209509977839674160L;
+
+ public final static String HELIX_ZOOKEEPER = ContainerProcessProperties.HELIX_ZOOKEEPER;
+ public final static String HELIX_CLUSTER = ContainerProcessProperties.HELIX_CLUSTER;
+ public final static String PROVIDER_METADATA = "provider.metadata";
+ public final static String PROVIDER_NAME = "provider.name";
+ public final static String CONTAINER_ID = "container.id";
+ public final static String YARN_RESOURCEMANAGER = "yarn.resourcemananger";
+ public final static String YARN_SCHEDULER = "yarn.scheduler";
+ public final static String YARN_USER = "yarn.user";
+ public final static String YARN_HDFS= "yarn.hdfs";
+
+ public boolean isValidMaster() {
+ return containsKey(HELIX_ZOOKEEPER) &&
+ containsKey(HELIX_CLUSTER) &&
+ containsKey(PROVIDER_METADATA) &&
+ containsKey(PROVIDER_NAME) &&
+ containsKey(YARN_RESOURCEMANAGER) &&
+ containsKey(YARN_SCHEDULER) &&
+ containsKey(YARN_USER) &&
+ containsKey(YARN_HDFS);
+ }
+
+ public boolean isValidContainer() {
+ return containsKey(HELIX_ZOOKEEPER) &&
+ containsKey(HELIX_CLUSTER) &&
+ containsKey(PROVIDER_METADATA) &&
+ containsKey(CONTAINER_ID);
+ }
+
+ public String getHelixZookeeper() {
+ return getProperty(HELIX_ZOOKEEPER);
+ }
+
+ public String getHelixCluster() {
+ return getProperty(HELIX_CLUSTER);
+ }
+
+ public String getProviderMetadata() {
+ return getProperty(PROVIDER_METADATA);
+ }
+
+ public String getProviderName() {
+ return getProperty(PROVIDER_NAME);
+ }
+
+ public String getContainerId() {
+ return getProperty(CONTAINER_ID);
+ }
+
+ public String getYarnResourceManager() {
+ return getProperty(YARN_RESOURCEMANAGER);
+ }
+
+ public String getYarnScheduler() {
+ return getProperty(YARN_SCHEDULER);
+ }
+
+ public String getYarnUser() {
+ return getProperty(YARN_USER);
+ }
+
+ public String getYarnHdfs() {
+ return getProperty(YARN_HDFS);
+ }
+
+ @Override
+ public Object get(Object key) {
+ Preconditions.checkState(containsKey(key));
+ return super.get(key);
+ }
+
+ @Override
+ public String getProperty(String key) {
+ Preconditions.checkState(containsKey(key));
+ return super.getProperty(key);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerData.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerData.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerData.java
new file mode 100644
index 0000000..d369a2d
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerData.java
@@ -0,0 +1,86 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+/**
+ * Container meta data for YARN-based containers. Reflect lifecycle of container
+ * from requesting, to bootstrapping, active operation and shutdown. Read and
+ * written by {@link YarnMasterProcess}, {@link YarnContainerProvider} and
+ * {@link YarnContainerService}. Also read by {@link YarnStatusProvider}.
+ * Typically stored in zookeeper
+ *
+ */
+class YarnContainerData {
+
+ static enum ContainerState {
+ ACQUIRE,
+ CONNECTING,
+ ACTIVE,
+ TEARDOWN,
+ FAILED,
+ HALTED,
+ FINALIZE
+ }
+
+ String id;
+ ContainerState state;
+ int yarnId;
+ String owner;
+ YarnContainerProcessProperties properties;
+
+ public YarnContainerData() {
+ // left blank
+ }
+
+ public YarnContainerData(String id, String owner, YarnContainerProcessProperties properties) {
+ this.id = id;
+ this.state = ContainerState.ACQUIRE;
+ this.yarnId = -1;
+ this.owner = owner;
+ this.properties = properties;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public YarnContainerData setId(String id) {
+ this.id = id;
+ return this;
+ }
+
+ public ContainerState getState() {
+ return state;
+ }
+
+ public YarnContainerData setState(ContainerState state) {
+ this.state = state;
+ return this;
+ }
+
+ public int getYarnId() {
+ return yarnId;
+ }
+
+ public YarnContainerData setYarnId(int yarnId) {
+ this.yarnId = yarnId;
+ return this;
+ }
+
+ public String getOwner() {
+ return owner;
+ }
+
+ public YarnContainerData setOwner(String owner) {
+ this.owner = owner;
+ return this;
+ }
+
+ public YarnContainerProcessProperties getProperties() {
+ return properties;
+ }
+
+ public YarnContainerData setProperties(YarnContainerProcessProperties properties) {
+ this.properties = properties;
+ return this;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProcess.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProcess.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProcess.java
new file mode 100644
index 0000000..2cad52d
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProcess.java
@@ -0,0 +1,53 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+import org.apache.helix.metamanager.container.ContainerProcess;
+import org.apache.log4j.Logger;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Host process for {@link ContainerProcess}es spawned by
+ * {@link YarnContainerProvider}. Configured via *.properties file in working
+ * directory. Corresponds to regular container in YARN and is managed jointly by
+ * the application master and the Helix participant. (Program entry point)
+ *
+ */
+class YarnContainerProcess {
+ static final Logger log = Logger.getLogger(YarnContainerProcess.class);
+
+ public static void main(String[] args) throws Exception {
+ log.trace("BEGIN YarnProcess.main()");
+
+ final YarnContainerProcessProperties properties = YarnUtils.createContainerProcessProperties(YarnUtils
+ .getPropertiesFromPath(YarnUtils.YARN_CONTAINER_PROPERTIES));
+ Preconditions.checkArgument(properties.isValid(), "container properties not valid: %s", properties.toString());
+
+ log.debug("Launching yarndata service");
+ final ZookeeperYarnDataProvider metaService = new ZookeeperYarnDataProvider(properties.getYarnData());
+ metaService.start();
+
+ log.debug("Launching yarn container service");
+ final YarnContainerService yarnService = new YarnContainerService();
+ yarnService.configure(properties);
+ yarnService.setYarnDataProvider(metaService);
+ yarnService.start();
+
+ log.debug("Installing shutdown hooks");
+ Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+ @Override
+ public void run() {
+ log.debug("Running shutdown hook");
+ yarnService.stop();
+ metaService.stop();
+ }
+ }));
+
+ System.out.println("Press ENTER to stop container process");
+ System.in.read();
+
+ log.debug("Stopping container services");
+ System.exit(0);
+
+ log.trace("END YarnProcess.main()");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProcessProperties.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProcessProperties.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProcessProperties.java
new file mode 100644
index 0000000..5277e2f
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProcessProperties.java
@@ -0,0 +1,40 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+import org.apache.helix.metamanager.container.ContainerProcessProperties;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Base configuration for {@link YarnContainerProcess}.
+ *
+ */
+public class YarnContainerProcessProperties extends ContainerProcessProperties {
+ /**
+ *
+ */
+ private static final long serialVersionUID = -2209509977839674160L;
+
+ public final static String YARNDATA = "yarndata";
+
+ public boolean isValid() {
+ return super.isValid() &&
+ containsKey(YARNDATA);
+ }
+
+ public String getYarnData() {
+ return getProperty(YARNDATA);
+ }
+
+ @Override
+ public Object get(Object key) {
+ Preconditions.checkState(containsKey(key));
+ return super.get(key);
+ }
+
+ @Override
+ public String getProperty(String key) {
+ Preconditions.checkState(containsKey(key));
+ return super.getProperty(key);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProvider.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProvider.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProvider.java
new file mode 100644
index 0000000..c229a26
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProvider.java
@@ -0,0 +1,143 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.helix.metamanager.ContainerProvider;
+import org.apache.helix.metamanager.ContainerProviderService;
+import org.apache.helix.metamanager.impl.yarn.YarnContainerData.ContainerState;
+import org.apache.log4j.Logger;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * {@link ContainerProvider} spawning YARN-based containers. Reads and writes
+ * meta data using {@link YarnDataProvider}. Works in a distributed setting, but
+ * typically requires access to zookeeper.
+ *
+ */
+class YarnContainerProvider implements ContainerProviderService {
+
+ static final Logger log = Logger.getLogger(YarnContainerProvider.class);
+
+ static final long POLL_INTERVAL = 1000;
+ static final long CONTAINER_TIMEOUT = 60000;
+
+ /*
+ * CONTAINERS
+ * A (A, READY)
+ * B (B, RUNNING)
+ */
+
+ final Object notifier = new Object();
+ final Map<String, Properties> types = new HashMap<String, Properties>();
+
+ ZookeeperYarnDataProvider yarnDataService;
+ YarnContainerProviderProcess yarnApp;
+ YarnContainerProviderProperties properties;
+
+ @Override
+ public void configure(Properties properties) throws Exception {
+ YarnContainerProviderProperties yarnProps = new YarnContainerProviderProperties();
+ yarnProps.putAll(properties);
+ configure(yarnProps);
+ }
+
+ private void configure(YarnContainerProviderProperties properties) {
+ this.properties = properties;
+
+ for(String containerType : properties.getContainers()) {
+ registerType(containerType, properties.getContainer(containerType));
+ }
+ }
+
+ @Override
+ public void start() throws Exception {
+ Preconditions.checkNotNull(properties);
+ Preconditions.checkState(properties.isValid(), "provider properties not valid: %s", properties);
+
+ log.debug("Starting yarn container provider service");
+ yarnDataService = new ZookeeperYarnDataProvider();
+ yarnDataService.configure(properties);
+ yarnDataService.start();
+ }
+
+ @Override
+ public void stop() throws Exception {
+ log.debug("Stopping yarn container provider service");
+ destroyAll();
+
+ if(yarnDataService != null) {
+ yarnDataService.stop();
+ yarnDataService = null;
+ }
+ }
+
+ @Override
+ public void create(final String id, final String type) throws Exception {
+ Preconditions.checkArgument(types.containsKey(type), "Container type '%s' is not configured", type);
+
+ YarnContainerProcessProperties containerProperties = YarnUtils.createContainerProcessProperties(types.get(type));
+
+ log.info(String.format("Running container '%s' (properties='%s')", id, containerProperties));
+
+ yarnDataService.create(new YarnContainerData(id, properties.getName(), containerProperties));
+ waitForState(id, ContainerState.ACTIVE);
+ }
+
+ @Override
+ public void destroy(final String id) throws Exception {
+ YarnContainerData meta = yarnDataService.read(id);
+
+ if(meta.state == ContainerState.ACTIVE) {
+ log.info(String.format("Destroying active container, going to teardown"));
+ yarnDataService.update(meta.setState(ContainerState.TEARDOWN));
+
+ } else if(meta.state == ContainerState.FAILED) {
+ log.info(String.format("Destroying failed container, going to teardown"));
+ yarnDataService.update(meta.setState(ContainerState.TEARDOWN));
+
+ } else if(meta.state == ContainerState.FINALIZE) {
+ log.info(String.format("Destroying finalized container, skipping"));
+
+ } else {
+ throw new IllegalStateException(String.format("Container '%s' must be active, failed or finalized", id));
+ }
+
+ waitForState(id, ContainerState.FINALIZE);
+ yarnDataService.delete(id);
+ }
+
+ @Override
+ public void destroyAll() {
+ try {
+ for(YarnContainerData meta : yarnDataService.readAll()) {
+ if(meta.owner.equals(properties.getName())) {
+ try { destroy(meta.id); } catch (Exception ignore) {}
+ }
+ }
+ } catch (Exception ignore) {
+ // ignore
+ }
+ }
+
+ void waitForState(String id, ContainerState state) throws Exception, InterruptedException, TimeoutException {
+ long limit = System.currentTimeMillis() + CONTAINER_TIMEOUT;
+ YarnContainerData meta = yarnDataService.read(id);
+ while(meta.state != state) {
+ if(System.currentTimeMillis() >= limit) {
+ throw new TimeoutException(String.format("Container '%s' failed to reach state '%s' (currently is '%s')", id, state, meta.state));
+ }
+ Thread.sleep(POLL_INTERVAL);
+ meta = yarnDataService.read(id);
+ }
+ }
+
+ void registerType(String name, Properties properties) {
+ log.debug(String.format("Registering container type '%s' (properties='%s')", name, properties));
+ types.put(name, properties);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProviderProcess.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProviderProcess.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProviderProcess.java
new file mode 100644
index 0000000..d883dce
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProviderProcess.java
@@ -0,0 +1,158 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+import java.io.File;
+import java.net.InetSocketAddress;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.ClientRMProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.helix.metamanager.Service;
+import org.apache.log4j.Logger;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Configurable and runnable service for {@link YarnContainerProvider}
+ *
+ */
+public class YarnContainerProviderProcess implements Service {
+
+ static final Logger log = Logger.getLogger(YarnContainerProviderProcess.class);
+
+ static String YARN_MASTER_COMMAND = "/bin/sh %s 1>%s/stdout 2>%s/stderr";
+
+ Configuration conf;
+ YarnRPC rpc;
+ ClientRMProtocol rmClient;
+ ApplicationId appId;
+ File propertiesFile;
+
+ YarnContainerProviderProperties properties;
+
+ @Override
+ public void configure(Properties properties) throws Exception {
+ configure(YarnUtils.createContainerProviderProperties(properties));
+ }
+
+ private void configure(YarnContainerProviderProperties properties) {
+ this.conf = new YarnConfiguration();
+ this.conf.set(YarnConfiguration.RM_ADDRESS, properties.getResourceManager());
+ this.conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, properties.getScheduler());
+ this.conf.set(FileSystem.FS_DEFAULT_NAME_KEY, properties.getHdfs());
+
+ this.rpc = YarnRPC.create(conf);
+
+ this.properties = properties;
+ }
+
+ @Override
+ public void start() throws Exception {
+ Preconditions.checkNotNull(properties);
+ Preconditions.checkState(properties.isValid());
+
+ connect();
+
+ String command = String.format(YARN_MASTER_COMMAND, YarnUtils.YARN_MASTER_PATH, ApplicationConstants.LOG_DIR_EXPANSION_VAR,
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+
+ log.info(String.format("Starting application '%s' provider '%s' (masterCommand='%s')", properties.getYarnData(), properties.getName(), command));
+
+ log.debug(String.format("Running master command \"%s\"", command));
+
+ // app id
+ GetNewApplicationRequest appRequest = Records.newRecord(GetNewApplicationRequest.class);
+ GetNewApplicationResponse appResponse = rmClient.getNewApplication(appRequest);
+
+ this.appId = appResponse.getApplicationId();
+
+ log.info(String.format("Acquired app id '%s' for '%s'", appId.toString(), properties.getName()));
+
+ // command
+ ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class);
+ launchContext.setCommands(Collections.singletonList(command));
+
+ // resource limit
+ Resource resource = Records.newRecord(Resource.class);
+ resource.setMemory(256); // TODO make dynamic
+ launchContext.setResource(resource);
+
+ // environment
+ Map<String, String> env = new HashMap<String, String>();
+ launchContext.setEnvironment(env);
+
+ // configuration
+ propertiesFile = YarnUtils.writePropertiesToTemp(properties);
+
+ // HDFS
+ final String namespace = appId.toString();
+ final Path masterArchive = YarnUtils.copyToHdfs(YarnUtils.YARN_MASTER_ARCHIVE_PATH, YarnUtils.YARN_MASTER_STAGING, namespace, conf);
+ final Path masterProperties = YarnUtils.copyToHdfs(propertiesFile.getCanonicalPath(), YarnUtils.YARN_MASTER_PROPERTIES, namespace, conf);
+ final Path containerArchive = YarnUtils.copyToHdfs(YarnUtils.YARN_CONTAINER_ARCHIVE_PATH, YarnUtils.YARN_CONTAINER_STAGING, namespace, conf);
+
+ // local resources
+ Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
+ localResources.put(YarnUtils.YARN_MASTER_DESTINATION, YarnUtils.createHdfsResource(masterArchive, LocalResourceType.ARCHIVE, conf));
+ localResources.put(YarnUtils.YARN_MASTER_PROPERTIES, YarnUtils.createHdfsResource(masterProperties, LocalResourceType.FILE, conf));
+ localResources.put(YarnUtils.YARN_CONTAINER_STAGING, YarnUtils.createHdfsResource(containerArchive, LocalResourceType.FILE, conf));
+
+ launchContext.setLocalResources(localResources);
+
+ // user
+ launchContext.setUser(properties.getUser());
+
+ // app submission
+ ApplicationSubmissionContext subContext = Records.newRecord(ApplicationSubmissionContext.class);
+ subContext.setApplicationId(appId);
+ subContext.setApplicationName(properties.getName());
+ subContext.setAMContainerSpec(launchContext);
+
+ SubmitApplicationRequest subRequest = Records.newRecord(SubmitApplicationRequest.class);
+ subRequest.setApplicationSubmissionContext(subContext);
+
+ log.info(String.format("Starting app id '%s'", appId.toString()));
+
+ rmClient.submitApplication(subRequest);
+
+ }
+
+ @Override
+ public void stop() throws YarnRemoteException {
+ log.info(String.format("Stopping app id '%s'", appId.toString()));
+ KillApplicationRequest killRequest = Records.newRecord(KillApplicationRequest.class);
+ killRequest.setApplicationId(appId);
+
+ rmClient.forceKillApplication(killRequest);
+
+ try { YarnUtils.destroyHdfsNamespace(appId.toString(), conf); } catch(Exception ignore) {}
+
+ propertiesFile.delete();
+ }
+
+ void connect() {
+ YarnConfiguration yarnConf = new YarnConfiguration(conf);
+ InetSocketAddress rmAddress = NetUtils.createSocketAddr(yarnConf.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS));
+ log.info("Connecting to ResourceManager at: " + rmAddress);
+ this.rmClient = ((ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class, rmAddress, conf));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProviderProperties.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProviderProperties.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProviderProperties.java
new file mode 100644
index 0000000..95ad0aa
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerProviderProperties.java
@@ -0,0 +1,64 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+import org.apache.helix.metamanager.provider.ProviderProperties;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Base configuration for {@link YarnContainerProviderProcess}
+ *
+ */
+public class YarnContainerProviderProperties extends ProviderProperties {
+ /**
+ *
+ */
+ private static final long serialVersionUID = -8853614843205587170L;
+
+ public final static String YARNDATA = "yarndata";
+ public final static String RESOURCEMANAGER = "resourcemananger";
+ public final static String SCHEDULER = "scheduler";
+ public final static String USER = "user";
+ public final static String HDFS = "hdfs";
+
+ public boolean isValid() {
+ return super.isValid() &&
+ containsKey(YARNDATA) &&
+ containsKey(RESOURCEMANAGER) &&
+ containsKey(SCHEDULER) &&
+ containsKey(USER) &&
+ containsKey(HDFS);
+ }
+
+ public String getYarnData() {
+ return getProperty(YARNDATA);
+ }
+
+ public String getResourceManager() {
+ return getProperty(RESOURCEMANAGER);
+ }
+
+ public String getScheduler() {
+ return getProperty(SCHEDULER);
+ }
+
+ public String getUser() {
+ return getProperty(USER);
+ }
+
+ public String getHdfs() {
+ return getProperty(HDFS);
+ }
+
+ @Override
+ public String getProperty(String key) {
+ Preconditions.checkState(containsKey(key));
+ return super.getProperty(key);
+ }
+
+ @Override
+ public Object get(Object key) {
+ Preconditions.checkState(containsKey(key));
+ return super.get(key);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/e38aa54b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerService.java
----------------------------------------------------------------------
diff --git a/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerService.java b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerService.java
new file mode 100644
index 0000000..804d6ed
--- /dev/null
+++ b/recipes/meta-cluster-manager/src/main/java/org/apache/helix/metamanager/impl/yarn/YarnContainerService.java
@@ -0,0 +1,156 @@
+package org.apache.helix.metamanager.impl.yarn;
+
+import java.io.File;
+import java.util.Properties;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.helix.metamanager.Service;
+import org.apache.helix.metamanager.container.ContainerProcess;
+import org.apache.helix.metamanager.container.ContainerProcessProperties;
+import org.apache.helix.metamanager.container.ContainerUtils;
+import org.apache.helix.metamanager.impl.yarn.YarnContainerData.ContainerState;
+import org.apache.log4j.Logger;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Configurable and runnable service for YARN-based containers. Continuously
+ * checks container meta data and process state and triggers state changes and
+ * container setup and shutdown.
+ *
+ */
+class YarnContainerService implements Service {
+ static final Logger log = Logger.getLogger(YarnContainerService.class);
+
+ static final long CONTAINERSERVICE_INTERVAL = 1000;
+
+ YarnContainerProcessProperties properties;
+
+ YarnDataProvider metaService;
+ ScheduledExecutorService executor;
+
+ ContainerProcess process;
+
+ @Override
+ public void configure(Properties properties) throws Exception {
+ Preconditions.checkNotNull(properties);
+ YarnContainerProcessProperties containerProperties = new YarnContainerProcessProperties();
+ containerProperties.putAll(properties);
+ Preconditions.checkArgument(containerProperties.isValid());
+
+ this.properties = containerProperties;
+ }
+
+ public void setYarnDataProvider(YarnDataProvider metaService) {
+ this.metaService = metaService;
+ }
+
+ @Override
+ public void start() {
+ Preconditions.checkNotNull(metaService);
+ Preconditions.checkNotNull(properties);
+ Preconditions.checkState(properties.isValid());
+
+ log.debug("starting yarn container service");
+
+ executor = Executors.newSingleThreadScheduledExecutor();
+ executor.scheduleAtFixedRate(new ContainerStatusService(), 0, CONTAINERSERVICE_INTERVAL, TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public void stop() {
+ log.debug("stopping yarn container service");
+
+ if (executor != null) {
+ executor.shutdown();
+ while (!executor.isTerminated()) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ // ignore
+ }
+ }
+ executor = null;
+ }
+
+ destroyLocalContainerNamespace();
+ }
+
+ class ContainerStatusService implements Runnable {
+ @Override
+ public void run() {
+ log.info("updating container status");
+
+ try {
+ if (!metaService.exists(properties.getName())) {
+ log.warn(String.format("YarnData for '%s' does not exist. Terminating yarn service.", properties.getName()));
+ process.stop();
+ stop();
+ }
+
+ YarnContainerData meta = metaService.read(properties.getName());
+
+ if (meta.state == ContainerState.CONNECTING) {
+ log.trace("container connecting");
+ try {
+ ContainerProcessProperties containerProperties = meta.getProperties();
+
+ containerProperties.setProperty(ContainerProcessProperties.CLUSTER, properties.getCluster());
+ containerProperties.setProperty(ContainerProcessProperties.ADDRESS, properties.getAddress());
+ containerProperties.setProperty(ContainerProcessProperties.NAME, properties.getName());
+
+ process = ContainerUtils.createProcess(containerProperties);
+ process.start();
+ } catch (Exception e) {
+ log.error("Failed to start participant, going to failed", e);
+ }
+
+ if (process.isActive()) {
+ log.trace("process active, activating container");
+ metaService.update(meta.setState(ContainerState.ACTIVE));
+
+ } else if (process.isFailed()) {
+ log.trace("process failed, failing container");
+ metaService.update(meta.setState(ContainerState.FAILED));
+
+ } else {
+ log.trace("process state unknown, failing container");
+ metaService.update(meta.setState(ContainerState.FAILED));
+ }
+ }
+
+ if (meta.state == ContainerState.ACTIVE) {
+ log.trace("container active");
+ if (process.isFailed()) {
+ log.trace("process failed, failing container");
+ metaService.update(meta.setState(ContainerState.FAILED));
+
+ } else if (!process.isActive()) {
+ log.trace("process not active, halting container");
+ process.stop();
+ metaService.update(meta.setState(ContainerState.HALTED));
+ }
+ }
+
+ if (meta.state == ContainerState.TEARDOWN) {
+ log.trace("container teardown");
+ process.stop();
+ metaService.update(meta.setState(ContainerState.HALTED));
+ }
+
+ } catch (Exception e) {
+ log.error(String.format("Error while updating container '%s' status", properties.getName()), e);
+ }
+ }
+ }
+
+ public static void destroyLocalContainerNamespace() {
+ log.info("cleaning up container directory");
+ FileUtils.deleteQuietly(new File(YarnUtils.YARN_CONTAINER_DESTINATION));
+ FileUtils.deleteQuietly(new File(YarnUtils.YARN_CONTAINER_PROPERTIES));
+ }
+
+}