You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by ka...@apache.org on 2014/07/10 19:05:00 UTC
[17/50] [abbrv] Moving packages around
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/Client.java
----------------------------------------------------------------------
diff --git a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/Client.java b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/Client.java
deleted file mode 100644
index 500df9c..0000000
--- a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/Client.java
+++ /dev/null
@@ -1,627 +0,0 @@
-package org.apache.helix.provisioning.yarn;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Vector;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.compress.archivers.ArchiveStreamFactory;
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
-import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.QueueACL;
-import org.apache.hadoop.yarn.api.records.QueueInfo;
-import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.client.api.YarnClientApplication;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.hadoop.yarn.util.Records;
-
-/**
- * Client for Distributed Shell application submission to YARN.
- * <p>
- * The distributed shell client allows an application master to be launched that in turn would run
- * the provided shell command on a set of containers.
- * </p>
- * <p>
- * This client is meant to act as an example on how to write yarn-based applications.
- * </p>
- * <p>
- * To submit an application, a client first needs to connect to the <code>ResourceManager</code> aka
- * ApplicationsManager or ASM via the {@link ApplicationClientProtocol}. The
- * {@link ApplicationClientProtocol} provides a way for the client to get access to cluster
- * information and to request for a new {@link ApplicationId}.
- * <p>
- * <p>
- * For the actual job submission, the client first has to create an
- * {@link ApplicationSubmissionContext}. The {@link ApplicationSubmissionContext} defines the
- * application details such as {@link ApplicationId} and application name, the priority assigned to
- * the application and the queue to which this application needs to be assigned. In addition to
- * this, the {@link ApplicationSubmissionContext} also defines the {@link ContainerLaunchContext}
- * which describes the <code>Container</code> with which the {@link ApplicationMaster} is launched.
- * </p>
- * <p>
- * The {@link ContainerLaunchContext} in this scenario defines the resources to be allocated for the
- * {@link ApplicationMaster}'s container, the local resources (jars, configuration files) to be made
- * available and the environment to be set for the {@link ApplicationMaster} and the commands to be
- * executed to run the {@link ApplicationMaster}.
- * <p>
- * <p>
- * Using the {@link ApplicationSubmissionContext}, the client submits the application to the
- * <code>ResourceManager</code> and then monitors the application by requesting the
- * <code>ResourceManager</code> for an {@link ApplicationReport} at regular time intervals. In case
- * of the application taking too long, the client kills the application by submitting a
- * {@link KillApplicationRequest} to the <code>ResourceManager</code>.
- * </p>
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class Client {
-
- private static final Log LOG = LogFactory.getLog(Client.class);
-
- // Configuration
- private Configuration conf;
- private YarnClient yarnClient;
- // Application master specific info to register a new Application with RM/ASM
- private String appName = "";
- // App master priority
- private int amPriority = 0;
- // Queue for App master
- private String amQueue = "";
- // Amt. of memory resource to request for to run the App Master
- private int amMemory = 1024;
-
- // Application master jar file
- private String appMasterArchive = "";
- // Main class to invoke application master
- private final String appMasterMainClass;
-
- private String appSpecFile = "";
-
- // No. of containers in which helix participants will be started
- private int numContainers = 1;
-
- // log4j.properties file
- // if available, add to local resources and set into classpath
- private String log4jPropFile = "";
-
- // Start time for client
- private final long clientStartTime = System.currentTimeMillis();
- // Timeout threshold for client. Kill app after time interval expires.
- private long clientTimeout = 600000;
-
- // Debug flag
- boolean debugFlag = false;
-
- // Command line options
- private Options opts;
-
- /**
- */
- public Client(Configuration conf) throws Exception {
- this("org.apache.helix.provisioning.yarn.HelixYarnApplicationMasterMain", conf);
- }
-
- Client(String appMasterMainClass, Configuration conf) {
- this.conf = conf;
- this.appMasterMainClass = appMasterMainClass;
- yarnClient = YarnClient.createYarnClient();
- yarnClient.init(conf);
- opts = new Options();
- opts.addOption("appName", true, "Application Name.");
- opts.addOption("priority", true, "Application Priority. Default 0");
- opts.addOption("queue", true, "RM Queue in which this application is to be submitted");
- opts.addOption("master_memory", true,
- "Amount of memory in MB to be requested to run the application master");
- opts.addOption("archive", true, "Archive file containing the app code");
- opts.addOption("appSpec", true, "Application specification");
- opts.addOption("num_containers", true,
- "No. of containers on which Helix Participants will be launched");
- opts.addOption("log_properties", true, "log4j.properties file");
- opts.addOption("debug", false, "Dump out debug information");
- opts.addOption("help", false, "Print usage");
-
- }
-
- /**
- */
- public Client() throws Exception {
- this(new YarnConfiguration());
- }
-
- /**
- * Helper function to print out usage
- */
- private void printUsage() {
- new HelpFormatter().printHelp("Client", opts);
- }
-
- /**
- * Parse command line options
- * @param args Parsed command line options
- * @return Whether the init was successful to run the client
- * @throws ParseException
- */
- public boolean init(String[] args) throws ParseException {
-
- CommandLine cliParser = new GnuParser().parse(opts, args);
-
- if (args.length == 0) {
- throw new IllegalArgumentException("No args specified for client to initialize");
- }
-
- if (cliParser.hasOption("help")) {
- printUsage();
- return false;
- }
-
- if (cliParser.hasOption("debug")) {
- debugFlag = true;
- }
-
- appName = cliParser.getOptionValue("appName");
- amPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0"));
- amQueue = cliParser.getOptionValue("queue", "default");
- amMemory = Integer.parseInt(cliParser.getOptionValue("master_memory", "1024"));
-
- if (amMemory < 0) {
- throw new IllegalArgumentException(
- "Invalid memory specified for application master, exiting." + " Specified memory="
- + amMemory);
- }
-
- if (!cliParser.hasOption("archive")) {
- throw new IllegalArgumentException("No archive file specified for application master");
- }
-
- appMasterArchive = cliParser.getOptionValue("archive");
-
- numContainers = Integer.parseInt(cliParser.getOptionValue("num_containers", "4"));
-
- log4jPropFile = cliParser.getOptionValue("log_properties", "");
-
- return true;
- }
-
- /**
- * Main run function for the client
- * @return true if application completed successfully
- * @throws IOException
- * @throws YarnException
- */
- public boolean run() throws IOException, YarnException {
-
- LOG.info("Running Client");
- yarnClient.start();
-
- YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics();
- LOG.info("Got Cluster metric info from ASM" + ", numNodeManagers="
- + clusterMetrics.getNumNodeManagers());
-
- List<NodeReport> clusterNodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
- LOG.info("Got Cluster node info from ASM");
- for (NodeReport node : clusterNodeReports) {
- LOG.info("Got node report from ASM for" + ", nodeId=" + node.getNodeId() + ", nodeAddress"
- + node.getHttpAddress() + ", nodeRackName" + node.getRackName() + ", nodeNumContainers"
- + node.getNumContainers());
- }
-
- QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue);
- LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueCurrentCapacity="
- + queueInfo.getCurrentCapacity() + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity()
- + ", queueApplicationCount=" + queueInfo.getApplications().size()
- + ", queueChildQueueCount=" + queueInfo.getChildQueues().size());
-
- List<QueueUserACLInfo> listAclInfo = yarnClient.getQueueAclsInfo();
- for (QueueUserACLInfo aclInfo : listAclInfo) {
- for (QueueACL userAcl : aclInfo.getUserAcls()) {
- LOG.info("User ACL Info for Queue" + ", queueName=" + aclInfo.getQueueName() + ", userAcl="
- + userAcl.name());
- }
- }
-
- // Get a new application id
- YarnClientApplication app = yarnClient.createApplication();
- GetNewApplicationResponse appResponse = app.getNewApplicationResponse();
- // TODO get min/max resource capabilities from RM and change memory ask if needed
- // If we do not have min/max, we may not be able to correctly request
- // the required resources from the RM for the app master
- // Memory ask has to be a multiple of min and less than max.
- // Dump out information about cluster capability as seen by the resource manager
- int maxMem = appResponse.getMaximumResourceCapability().getMemory();
- LOG.info("Max mem capabililty of resources in this cluster " + maxMem);
-
- // A resource ask cannot exceed the max.
- if (amMemory > maxMem) {
- LOG.info("AM memory specified above max threshold of cluster. Using max value."
- + ", specified=" + amMemory + ", max=" + maxMem);
- amMemory = maxMem;
- }
-
- // set the application name
- ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
- ApplicationId appId = appContext.getApplicationId();
- appContext.setApplicationName(appName);
-
- // Set up the container launch context for the application master
- ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
-
- // set local resources for the application master
- // local files or archives as needed
- // In this scenario, the jar file for the application master is part of the local resources
- Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
-
- LOG.info("Copy App archive file from local filesystem and add to local environment");
- // Copy the application master jar to the filesystem
- // Create a local resource to point to the destination jar path
- FileSystem fs = FileSystem.get(conf);
- Path src = new Path(appMasterArchive);
- String pathSuffix = appName + "/" + appId.getId() + "/app-pkg.tar";
- Path dst = new Path(fs.getHomeDirectory(), pathSuffix);
- fs.copyFromLocalFile(false, true, src, dst);
- FileStatus destStatus = fs.getFileStatus(dst);
- LocalResource amJarRsrc = Records.newRecord(LocalResource.class);
-
- // Set the type of resource - file or archive
- // archives are untarred at destination
- // we don't need the jar file to be untarred for now
- amJarRsrc.setType(LocalResourceType.ARCHIVE);
- // Set visibility of the resource
- // Setting to most private option
- amJarRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
- // Set the resource to be copied over
- amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(dst));
- // Set timestamp and length of file so that the framework
- // can do basic sanity checks for the local resource
- // after it has been copied over to ensure it is the same
- // resource the client intended to use with the application
- amJarRsrc.setTimestamp(destStatus.getModificationTime());
- amJarRsrc.setSize(destStatus.getLen());
- localResources.put("app-pkg", amJarRsrc);
-
- Path localAppSpec = new Path(appSpecFile);
- pathSuffix = appName + "/" + appId.getId() + "/app-spec.yaml";
- Path dstAppSpec = new Path(fs.getHomeDirectory(), pathSuffix);
- fs.copyFromLocalFile(false, true, localAppSpec, dstAppSpec);
- destStatus = fs.getFileStatus(dst);
- LocalResource appSpecResource = Records.newRecord(LocalResource.class);
-
- appSpecResource.setType(LocalResourceType.FILE);
- appSpecResource.setVisibility(LocalResourceVisibility.APPLICATION);
- appSpecResource.setResource(ConverterUtils.getYarnUrlFromPath(dstAppSpec));
- appSpecResource.setTimestamp(destStatus.getModificationTime());
- appSpecResource.setSize(destStatus.getLen());
- localResources.put("app-spec", appSpecResource);
-
- // Set the log4j properties if needed
- if (!log4jPropFile.isEmpty()) {
- Path log4jSrc = new Path(log4jPropFile);
- Path log4jDst = new Path(fs.getHomeDirectory(), "log4j.props");
- fs.copyFromLocalFile(false, true, log4jSrc, log4jDst);
- FileStatus log4jFileStatus = fs.getFileStatus(log4jDst);
- LocalResource log4jRsrc = Records.newRecord(LocalResource.class);
- log4jRsrc.setType(LocalResourceType.FILE);
- log4jRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
- log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri()));
- log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime());
- log4jRsrc.setSize(log4jFileStatus.getLen());
- localResources.put("log4j.properties", log4jRsrc);
- }
-
- // Set local resource info into app master container launch context
- amContainer.setLocalResources(localResources);
-
- // Set the necessary security tokens as needed
- // amContainer.setContainerTokens(containerToken);
-
- // Add AppMaster.jar location to classpath
- // At some point we should not be required to add
- // the hadoop specific classpaths to the env.
- // It should be provided out of the box.
- // For now setting all required classpaths including
- // the classpath to "." for the application jar
- StringBuilder classPathEnv =
- new StringBuilder(Environment.CLASSPATH.$()).append(File.pathSeparatorChar).append("./*");
- StringBuilder appClassPathEnv = new StringBuilder();
- // put the jar files under the archive in the classpath
- try {
- final InputStream is = new FileInputStream(appMasterArchive);
- final TarArchiveInputStream debInputStream =
- (TarArchiveInputStream) new ArchiveStreamFactory().createArchiveInputStream("tar", is);
- TarArchiveEntry entry = null;
- while ((entry = (TarArchiveEntry) debInputStream.getNextEntry()) != null) {
- if (entry.isFile()) {
- appClassPathEnv.append(File.pathSeparatorChar);
- appClassPathEnv.append("./app-pkg/" + entry.getName());
- }
- }
- debInputStream.close();
-
- } catch (Exception e) {
- LOG.error("Unable to read archive file:" + appMasterArchive, e);
- }
- classPathEnv.append(appClassPathEnv);
- for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
- YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
- classPathEnv.append(File.pathSeparatorChar);
- classPathEnv.append(c.trim());
- }
- classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties");
-
- // add the runtime classpath needed for tests to work
- if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
- classPathEnv.append(':');
- classPathEnv.append(System.getProperty("java.class.path"));
- }
- System.out.println("classpath" + classPathEnv.toString());
- // Set the env variables to be setup in the env where the application master will be run
- LOG.info("Set the environment for the application master");
- Map<String, String> env = new HashMap<String, String>();
- env.put("app_pkg_path", fs.getHomeDirectory() + "/" + appName + "/" + appId.getId()
- + "/app-pkg.tar");
- env.put("appName", appName);
- env.put("appId", "" + appId.getId());
- env.put("CLASSPATH", classPathEnv.toString());
- env.put("appClasspath", appClassPathEnv.toString());
- env.put("containerParticipantMainClass",
- "org.apache.helix.provisioning.yarn.ParticipantLauncher");
- amContainer.setEnvironment(env);
-
- // Set the necessary command to execute the application master
- Vector<CharSequence> vargs = new Vector<CharSequence>(30);
-
- // Set java executable command
- LOG.info("Setting up app master command");
- vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
- // Set Xmx based on am memory size
- vargs.add("-Xmx" + amMemory + "m");
- // Set class name
- vargs.add(appMasterMainClass);
- // Set params for Application Master
- vargs.add("--num_containers " + String.valueOf(numContainers));
-
- if (debugFlag) {
- vargs.add("--debug");
- }
-
- vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout");
- vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr");
-
- // Get final commmand
- StringBuilder command = new StringBuilder();
- for (CharSequence str : vargs) {
- command.append(str).append(" ");
- }
-
- LOG.info("Completed setting up app master command " + command.toString());
- List<String> commands = new ArrayList<String>();
- commands.add(command.toString());
- amContainer.setCommands(commands);
-
- // Set up resource type requirements
- // For now, only memory is supported so we set memory requirements
- Resource capability = Records.newRecord(Resource.class);
- capability.setMemory(amMemory);
- appContext.setResource(capability);
-
- // Service data is a binary blob that can be passed to the application
- // Not needed in this scenario
- // amContainer.setServiceData(serviceData);
-
- // Setup security tokens
- if (UserGroupInformation.isSecurityEnabled()) {
- Credentials credentials = new Credentials();
- String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL);
- if (tokenRenewer == null || tokenRenewer.length() == 0) {
- throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer");
- }
-
- // For now, only getting tokens for the default file-system.
- final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials);
- if (tokens != null) {
- for (Token<?> token : tokens) {
- LOG.info("Got dt for " + fs.getUri() + "; " + token);
- }
- }
- DataOutputBuffer dob = new DataOutputBuffer();
- credentials.writeTokenStorageToStream(dob);
- ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
- amContainer.setTokens(fsTokens);
- }
-
- appContext.setAMContainerSpec(amContainer);
-
- // Set the priority for the application master
- Priority pri = Records.newRecord(Priority.class);
- // TODO - what is the range for priority? how to decide?
- pri.setPriority(amPriority);
- appContext.setPriority(pri);
-
- // Set the queue to which this application is to be submitted in the RM
- appContext.setQueue(amQueue);
-
- // Submit the application to the applications manager
- // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest);
- // Ignore the response as either a valid response object is returned on success
- // or an exception thrown to denote some form of a failure
- LOG.info("Submitting application to ASM");
-
- yarnClient.submitApplication(appContext);
-
- // TODO
- // Try submitting the same request again
- // app submission failure?
-
- // Monitor the application
- return monitorApplication(appId);
-
- }
-
- /**
- * Monitor the submitted application for completion.
- * Kill application if time expires.
- * @param appId Application Id of application to be monitored
- * @return true if application completed successfully
- * @throws YarnException
- * @throws IOException
- */
- private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {
-
- while (true) {
-
- // Check app status every 10 second.
- try {
- Thread.sleep(10000);
- } catch (InterruptedException e) {
- LOG.debug("Thread sleep in monitoring loop interrupted");
- }
-
- // Get application report for the appId we are interested in
- ApplicationReport report = yarnClient.getApplicationReport(appId);
-
- LOG.info("Got application report from ASM for" + ", appId=" + appId.getId()
- + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics="
- + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue="
- + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime="
- + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString()
- + ", distributedFinalState=" + report.getFinalApplicationStatus().toString()
- + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser());
-
- YarnApplicationState state = report.getYarnApplicationState();
- FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
- if (YarnApplicationState.FINISHED == state) {
- if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
- LOG.info("Application has completed successfully. Breaking monitoring loop");
- return true;
- } else {
- LOG.info("Application did finished unsuccessfully." + " YarnState=" + state.toString()
- + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
- return false;
- }
- } else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
- LOG.info("Application did not finish." + " YarnState=" + state.toString()
- + ", DSFinalStatus=" + dsStatus.toString() + ". Breaking monitoring loop");
- return false;
- }
-
- /*
- * if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
- * LOG.info("Reached client specified timeout for application. Killing application");
- * forceKillApplication(appId);
- * return false;
- * }
- */
- }
-
- }
-
- /**
- * Kill a submitted application by sending a call to the ASM
- * @param appId Application Id to be killed.
- * @throws YarnException
- * @throws IOException
- */
- private void forceKillApplication(ApplicationId appId) throws YarnException, IOException {
- // TODO clarify whether multiple jobs with the same app id can be submitted and be running at
- // the same time.
- // If yes, can we kill a particular attempt only?
-
- // Response can be ignored as it is non-null on success or
- // throws an exception in case of failures
- // yarnClient.killApplication(appId);
- }
-
- /**
- * @param args Command line arguments
- */
- public static void main(String[] args) {
- boolean result = false;
- try {
- Client client = new Client();
- LOG.info("Initializing Client");
- try {
- boolean doRun = client.init(args);
- if (!doRun) {
- System.exit(0);
- }
- } catch (IllegalArgumentException e) {
- System.err.println(e.getLocalizedMessage());
- client.printUsage();
- System.exit(-1);
- }
- result = client.run();
- } catch (Throwable t) {
- LOG.fatal("Error running CLient", t);
- System.exit(1);
- }
- if (result) {
- LOG.info("Application completed successfully");
- System.exit(0);
- }
- LOG.error("Application failed to complete successfully");
- System.exit(2);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ContainerAskResponse.java
----------------------------------------------------------------------
diff --git a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ContainerAskResponse.java b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ContainerAskResponse.java
deleted file mode 100644
index c570932..0000000
--- a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ContainerAskResponse.java
+++ /dev/null
@@ -1,17 +0,0 @@
-package org.apache.helix.provisioning.yarn;
-
-import org.apache.hadoop.yarn.api.records.Container;
-
-public class ContainerAskResponse {
-
- Container container;
-
- public Container getContainer() {
- return container;
- }
-
- public void setContainer(Container container) {
- this.container = container;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ContainerLaunchResponse.java
----------------------------------------------------------------------
diff --git a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ContainerLaunchResponse.java b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ContainerLaunchResponse.java
deleted file mode 100644
index c91cb93..0000000
--- a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ContainerLaunchResponse.java
+++ /dev/null
@@ -1,5 +0,0 @@
-package org.apache.helix.provisioning.yarn;
-
-public class ContainerLaunchResponse {
-
-}
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ContainerReleaseResponse.java
----------------------------------------------------------------------
diff --git a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ContainerReleaseResponse.java b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ContainerReleaseResponse.java
deleted file mode 100644
index 77d50ba..0000000
--- a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ContainerReleaseResponse.java
+++ /dev/null
@@ -1,5 +0,0 @@
-package org.apache.helix.provisioning.yarn;
-
-public class ContainerReleaseResponse {
-
-}
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ContainerStopResponse.java
----------------------------------------------------------------------
diff --git a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ContainerStopResponse.java b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ContainerStopResponse.java
deleted file mode 100644
index 4c0022a..0000000
--- a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ContainerStopResponse.java
+++ /dev/null
@@ -1,5 +0,0 @@
-package org.apache.helix.provisioning.yarn;
-
-public class ContainerStopResponse {
-
-}
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/DSConstants.java
----------------------------------------------------------------------
diff --git a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/DSConstants.java b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/DSConstants.java
deleted file mode 100644
index a9fdf3d..0000000
--- a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/DSConstants.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.helix.provisioning.yarn;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * Constants used in both Client and Application Master
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class DSConstants {
-
- /**
- * Environment key name pointing to the shell script's location
- */
- public static final String DISTRIBUTEDSHELLSCRIPTLOCATION = "DISTRIBUTEDSHELLSCRIPTLOCATION";
-
- /**
- * Environment key name denoting the file timestamp for the shell script.
- * Used to validate the local resource.
- */
- public static final String DISTRIBUTEDSHELLSCRIPTTIMESTAMP = "DISTRIBUTEDSHELLSCRIPTTIMESTAMP";
-
- /**
- * Environment key name denoting the file content length for the shell script.
- * Used to validate the local resource.
- */
- public static final String DISTRIBUTEDSHELLSCRIPTLEN = "DISTRIBUTEDSHELLSCRIPTLEN";
-}
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/FixedTargetProvider.java
----------------------------------------------------------------------
diff --git a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/FixedTargetProvider.java b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/FixedTargetProvider.java
new file mode 100644
index 0000000..83ad461
--- /dev/null
+++ b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/FixedTargetProvider.java
@@ -0,0 +1,20 @@
+package org.apache.helix.provisioning.yarn;
+
+import java.util.Collection;
+
+import org.apache.helix.api.Cluster;
+import org.apache.helix.api.Participant;
+import org.apache.helix.api.id.ResourceId;
+import org.apache.helix.controller.provisioner.TargetProvider;
+import org.apache.helix.controller.provisioner.TargetProviderResponse;
+
+public class FixedTargetProvider implements TargetProvider {
+
+ @Override
+ public TargetProviderResponse evaluateExistingContainers(Cluster cluster, ResourceId resourceId,
+ Collection<Participant> participants) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/GenericApplicationMaster.java
----------------------------------------------------------------------
diff --git a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/GenericApplicationMaster.java b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/GenericApplicationMaster.java
index 79eb402..346af4b 100644
--- a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/GenericApplicationMaster.java
+++ b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/GenericApplicationMaster.java
@@ -56,6 +56,10 @@ import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
+import org.apache.helix.provisioning.ContainerAskResponse;
+import org.apache.helix.provisioning.ContainerLaunchResponse;
+import org.apache.helix.provisioning.ContainerReleaseResponse;
+import org.apache.helix.provisioning.ContainerStopResponse;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnApplicationMasterMain.java
----------------------------------------------------------------------
diff --git a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnApplicationMasterMain.java b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnApplicationMasterMain.java
deleted file mode 100644
index 5884a35..0000000
--- a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnApplicationMasterMain.java
+++ /dev/null
@@ -1,159 +0,0 @@
-package org.apache.helix.provisioning.yarn;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Map;
-
-import org.I0Itec.zkclient.IDefaultNameSpace;
-import org.I0Itec.zkclient.ZkClient;
-import org.I0Itec.zkclient.ZkServer;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.helix.HelixController;
-import org.apache.helix.api.accessor.ClusterAccessor;
-import org.apache.helix.api.config.ClusterConfig;
-import org.apache.helix.api.config.ResourceConfig;
-import org.apache.helix.api.id.ClusterId;
-import org.apache.helix.api.id.ControllerId;
-import org.apache.helix.api.id.ResourceId;
-import org.apache.helix.controller.provisioner.ProvisionerConfig;
-import org.apache.helix.controller.rebalancer.config.FullAutoRebalancerConfig;
-import org.apache.helix.controller.rebalancer.config.RebalancerConfig;
-import org.apache.helix.manager.zk.ZkHelixConnection;
-import org.apache.helix.model.StateModelDefinition;
-import org.apache.helix.tools.StateModelConfigGenerator;
-import org.apache.log4j.Logger;
-
-/**
- * This will <br/>
- * <ul>
- * <li>start zookeeper automatically</li>
- * <li>create the cluster</li>
- * <li>set up resource(s)</li>
- * <li>start helix controller</li>
- * </ul>
- */
-public class HelixYarnApplicationMasterMain {
- public static Logger LOG = Logger.getLogger(HelixYarnApplicationMasterMain.class);
-
- @SuppressWarnings("unchecked")
- public static void main(String[] args) throws Exception{
- Map<String, String> env = System.getenv();
- LOG.info("Starting app master with the following environment variables");
- for (String key : env.keySet()) {
- LOG.info(key + "\t\t=" + env.get(key));
- }
- int numContainers = 1;
-
- Options opts;
- opts = new Options();
- opts.addOption("num_containers", true, "Number of containers");
- try {
- CommandLine cliParser = new GnuParser().parse(opts, args);
- numContainers = Integer.parseInt(cliParser.getOptionValue("num_containers"));
- } catch (Exception e) {
- LOG.error("Error parsing input arguments" + Arrays.toString(args), e);
- }
-
- // START ZOOKEEPER
- String dataDir = "dataDir";
- String logDir = "logDir";
- IDefaultNameSpace defaultNameSpace = new IDefaultNameSpace() {
-
- @Override
- public void createDefaultNameSpace(ZkClient zkClient) {
-
- }
- };
- try {
- FileUtils.deleteDirectory(new File(dataDir));
- FileUtils.deleteDirectory(new File(logDir));
- } catch (IOException e) {
- LOG.error(e);
- }
-
- final ZkServer server = new ZkServer(dataDir, logDir, defaultNameSpace);
- server.start();
-
- // start
- AppMasterConfig appMasterConfig = new AppMasterConfig();
- String containerIdStr = appMasterConfig.getContainerId();
- ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
- ApplicationAttemptId appAttemptID = containerId.getApplicationAttemptId();
-
- String configFile = AppMasterConfig.AppEnvironment.APP_SPEC_FILE.toString();
- String className = appMasterConfig.getApplicationSpecFactory();
-
- GenericApplicationMaster genericApplicationMaster = new GenericApplicationMaster(appAttemptID);
- try {
- genericApplicationMaster.start();
- } catch (Exception e) {
- LOG.error("Unable to start application master: ", e);
- }
- ApplicationSpecFactory factory = HelixYarnUtil.createInstance(className);
- YarnProvisioner.applicationMaster = genericApplicationMaster;
- YarnProvisioner.applicationMasterConfig = appMasterConfig;
- ApplicationSpec applicationSpec = factory.fromYaml(new FileInputStream(configFile));
- YarnProvisioner.applicationSpec = applicationSpec;
- String zkAddress = appMasterConfig.getZKAddress();
- String clusterName = appMasterConfig.getAppName();
-
- // CREATE CLUSTER and setup the resources
- // connect
- ZkHelixConnection connection = new ZkHelixConnection(zkAddress);
- connection.connect();
-
- // create the cluster
- ClusterId clusterId = ClusterId.from(clusterName);
- ClusterAccessor clusterAccessor = connection.createClusterAccessor(clusterId);
- StateModelDefinition statelessService =
- new StateModelDefinition(StateModelConfigGenerator.generateConfigForStatelessService());
- clusterAccessor.createCluster(new ClusterConfig.Builder(clusterId).addStateModelDefinition(
- statelessService).build());
- for (String service : applicationSpec.getServices()) {
- String resourceName = service;
- // add the resource with the local provisioner
- ResourceId resourceId = ResourceId.from(resourceName);
- YarnProvisionerConfig provisionerConfig = new YarnProvisionerConfig(resourceId);
- ServiceConfig serviceConfig = applicationSpec.getServiceConfig(resourceName);
- provisionerConfig.setNumContainers(serviceConfig.getIntField("num_containers", 1));
- serviceConfig.setSimpleField("service_name", service);
- FullAutoRebalancerConfig.Builder rebalancerConfigBuilder =
- new FullAutoRebalancerConfig.Builder(resourceId);
- RebalancerConfig rebalancerConfig =
- rebalancerConfigBuilder.stateModelDefId(statelessService.getStateModelDefId())//
- .build();
- ResourceConfig.Builder resourceConfigBuilder =
- new ResourceConfig.Builder(ResourceId.from(resourceName));
- ResourceConfig resourceConfig = resourceConfigBuilder.provisionerConfig(provisionerConfig) //
- .rebalancerConfig(rebalancerConfig) //
- .userConfig(serviceConfig) //
- .build();
- clusterAccessor.addResourceToCluster(resourceConfig);
- }
- // start controller
- ControllerId controllerId = ControllerId.from("controller1");
- HelixController controller = connection.createController(clusterId, controllerId);
- controller.start();
-
- Thread shutdownhook = new Thread(new Runnable() {
- @Override
- public void run() {
- server.shutdown();
- }
- });
- Runtime.getRuntime().addShutdownHook(shutdownhook);
- Thread.sleep(10000);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnUtil.java
----------------------------------------------------------------------
diff --git a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnUtil.java b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnUtil.java
deleted file mode 100644
index ad606ba..0000000
--- a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/HelixYarnUtil.java
+++ /dev/null
@@ -1,42 +0,0 @@
-package org.apache.helix.provisioning.yarn;
-
-import org.apache.log4j.Logger;
-
-public class HelixYarnUtil {
- private static Logger LOG = Logger.getLogger(HelixYarnUtil.class);
-
- @SuppressWarnings("unchecked")
- public static <T extends ApplicationSpecFactory> T createInstance(String className) {
- Class<ApplicationSpecFactory> factoryClazz = null;
- {
- try {
- factoryClazz =
- (Class<ApplicationSpecFactory>) Thread.currentThread().getContextClassLoader()
- .loadClass(className);
- } catch (ClassNotFoundException e) {
- try {
- factoryClazz =
- (Class<ApplicationSpecFactory>) ClassLoader.getSystemClassLoader().loadClass(
- className);
- } catch (ClassNotFoundException e1) {
- try {
- factoryClazz = (Class<ApplicationSpecFactory>) Class.forName(className);
- } catch (ClassNotFoundException e2) {
-
- }
- }
- }
- }
- System.out.println(System.getProperty("java.class.path"));
- if (factoryClazz == null) {
- LOG.error("Unable to find class:" + className);
- }
- ApplicationSpecFactory factory = null;
- try {
- factory = factoryClazz.newInstance();
- } catch (Exception e) {
- LOG.error("Unable to create instance of class: " + className, e);
- }
- return (T) factory;
- }
-}
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/NMCallbackHandler.java
----------------------------------------------------------------------
diff --git a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/NMCallbackHandler.java b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/NMCallbackHandler.java
index 1566c28..f7c3a9f 100644
--- a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/NMCallbackHandler.java
+++ b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/NMCallbackHandler.java
@@ -9,6 +9,8 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
+import org.apache.helix.provisioning.ContainerLaunchResponse;
+import org.apache.helix.provisioning.ContainerStopResponse;
import org.apache.log4j.Logger;
import com.google.common.annotations.VisibleForTesting;
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ParticipantLauncher.java
----------------------------------------------------------------------
diff --git a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ParticipantLauncher.java b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ParticipantLauncher.java
deleted file mode 100644
index 1a21a71..0000000
--- a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ParticipantLauncher.java
+++ /dev/null
@@ -1,135 +0,0 @@
-package org.apache.helix.provisioning.yarn;
-
-import java.util.Arrays;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.Options;
-import org.apache.helix.HelixConnection;
-import org.apache.helix.NotificationContext;
-import org.apache.helix.api.id.ClusterId;
-import org.apache.helix.api.id.ParticipantId;
-import org.apache.helix.manager.zk.AbstractParticipantService;
-import org.apache.helix.manager.zk.ZkHelixConnection;
-import org.apache.helix.messaging.handling.HelixTaskResult;
-import org.apache.helix.messaging.handling.MessageHandler;
-import org.apache.helix.messaging.handling.MessageHandlerFactory;
-import org.apache.helix.model.Message;
-import org.apache.helix.model.Message.MessageType;
-import org.apache.log4j.Logger;
-
-/**
- * Main class that invokes the Participant Api
- */
-public class ParticipantLauncher {
- private static Logger LOG = Logger.getLogger(ParticipantLauncher.class);
-
- public static void main(String[] args) {
-
- System.out.println("Starting Helix Participant: " + Arrays.toString(args));
- Options opts;
- opts = new Options();
- opts.addOption("cluster", true, "Cluster name, default app name");
- opts.addOption("participantId", true, "Participant Id");
- opts.addOption("zkAddress", true, "Zookeeper address");
- opts.addOption("participantClass", true, "Participant service class");
- try {
- CommandLine cliParser = new GnuParser().parse(opts, args);
- String zkAddress = cliParser.getOptionValue("zkAddress");
- final HelixConnection connection = new ZkHelixConnection(zkAddress);
- connection.connect();
- ClusterId clusterId = ClusterId.from(cliParser.getOptionValue("cluster"));
- ParticipantId participantId = ParticipantId.from(cliParser.getOptionValue("participantId"));
- String participantClass = cliParser.getOptionValue("participantClass");
- @SuppressWarnings("unchecked")
- Class<? extends AbstractParticipantService> clazz =
- (Class<? extends AbstractParticipantService>) Class.forName(participantClass);
- final AbstractParticipantService containerParticipant =
- clazz.getConstructor(HelixConnection.class, ClusterId.class, ParticipantId.class)
- .newInstance(connection, clusterId, participantId);
- containerParticipant.startAsync();
- containerParticipant.awaitRunning(60, TimeUnit.SECONDS);
- containerParticipant
- .getParticipant()
- .getMessagingService()
- .registerMessageHandlerFactory(MessageType.SHUTDOWN.toString(),
- new ShutdownMessageHandlerFactory(containerParticipant, connection));
- Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
- @Override
- public void run() {
- LOG.info("Received a shutdown signal. Stopping participant");
- containerParticipant.stopAsync();
- containerParticipant.awaitTerminated();
- connection.disconnect();
- }
- }) {
-
- });
- Thread.currentThread().join();
- } catch (Exception e) {
- e.printStackTrace();
- System.out.println("Failed to start Helix participant" + e);
- // System.exit(1);
- }
- try {
- Thread.currentThread().join();
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
-
- }
-
- public static class ShutdownMessageHandlerFactory implements MessageHandlerFactory {
- private final AbstractParticipantService _service;
- private final HelixConnection _connection;
-
- public ShutdownMessageHandlerFactory(AbstractParticipantService service,
- HelixConnection connection) {
- _service = service;
- _connection = connection;
- }
-
- @Override
- public MessageHandler createHandler(Message message, NotificationContext context) {
- return new ShutdownMessageHandler(_service, _connection, message, context);
- }
-
- @Override
- public String getMessageType() {
- return MessageType.SHUTDOWN.toString();
- }
-
- @Override
- public void reset() {
- }
-
- }
-
- public static class ShutdownMessageHandler extends MessageHandler {
- private final AbstractParticipantService _service;
- private final HelixConnection _connection;
-
- public ShutdownMessageHandler(AbstractParticipantService service, HelixConnection connection,
- Message message, NotificationContext context) {
- super(message, context);
- _service = service;
- _connection = connection;
- }
-
- @Override
- public HelixTaskResult handleMessage() throws InterruptedException {
- LOG.info("Received a shutdown message. Trying to shut down.");
- _service.stopAsync();
- _service.awaitTerminated();
- _connection.disconnect();
- System.exit(1);
- return null;
- }
-
- @Override
- public void onError(Exception e, ErrorCode code, ErrorType type) {
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/RMCallbackHandler.java
----------------------------------------------------------------------
diff --git a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/RMCallbackHandler.java b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/RMCallbackHandler.java
index 8612d3a..ced1431 100644
--- a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/RMCallbackHandler.java
+++ b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/RMCallbackHandler.java
@@ -11,6 +11,9 @@ import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.helix.provisioning.ContainerAskResponse;
+import org.apache.helix.provisioning.ContainerReleaseResponse;
+import org.apache.helix.provisioning.ContainerStopResponse;
import com.google.common.util.concurrent.SettableFuture;
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ServiceConfig.java
----------------------------------------------------------------------
diff --git a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ServiceConfig.java b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ServiceConfig.java
deleted file mode 100644
index 87b5f12..0000000
--- a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/ServiceConfig.java
+++ /dev/null
@@ -1,17 +0,0 @@
-package org.apache.helix.provisioning.yarn;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.helix.api.Scope;
-import org.apache.helix.api.config.UserConfig;
-import org.apache.helix.api.id.ResourceId;
-
-public class ServiceConfig extends UserConfig{
- public Map<String, String> config = new HashMap<String, String>();
-
- public ServiceConfig(Scope<ResourceId> scope) {
- super(scope);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/TaskConfig.java
----------------------------------------------------------------------
diff --git a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/TaskConfig.java b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/TaskConfig.java
deleted file mode 100644
index 0b500a9..0000000
--- a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/TaskConfig.java
+++ /dev/null
@@ -1,13 +0,0 @@
-package org.apache.helix.provisioning.yarn;
-
-import java.util.HashMap;
-import java.util.Map;
-
-
-public class TaskConfig {
- public Map<String, String> config = new HashMap<String, String>();
-
- public String getValue(String key) {
- return (config != null ? config.get(key) : null);
- }
-}
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/YarnProvisioner.java
----------------------------------------------------------------------
diff --git a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/YarnProvisioner.java b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/YarnProvisioner.java
index 2d6e306..833efa5 100644
--- a/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/YarnProvisioner.java
+++ b/helix-provisioning/src/main/java/org/apache/helix/provisioning/yarn/YarnProvisioner.java
@@ -44,6 +44,12 @@ import org.apache.helix.controller.provisioner.Provisioner;
import org.apache.helix.controller.provisioner.TargetProvider;
import org.apache.helix.controller.provisioner.TargetProviderResponse;
import org.apache.helix.model.InstanceConfig;
+import org.apache.helix.provisioning.ApplicationSpec;
+import org.apache.helix.provisioning.ContainerAskResponse;
+import org.apache.helix.provisioning.ContainerLaunchResponse;
+import org.apache.helix.provisioning.ContainerReleaseResponse;
+import org.apache.helix.provisioning.ContainerStopResponse;
+import org.apache.helix.provisioning.ParticipantLauncher;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/recipes/helloworld-provisioning-yarn/pom.xml
----------------------------------------------------------------------
diff --git a/recipes/helloworld-provisioning-yarn/pom.xml b/recipes/helloworld-provisioning-yarn/pom.xml
new file mode 100644
index 0000000..4cef9a7
--- /dev/null
+++ b/recipes/helloworld-provisioning-yarn/pom.xml
@@ -0,0 +1,159 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.helix.recipes</groupId>
+ <artifactId>recipes</artifactId>
+ <version>0.7.1-incubating-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>helloworld-provisioning-yarn</artifactId>
+ <packaging>bundle</packaging>
+ <name>Apache Helix :: Recipes :: Provisioning :: YARN :: Hello World</name>
+
+ <properties>
+ <osgi.import>
+ org.apache.helix*,
+ org.apache.log4j,
+ *
+ </osgi.import>
+ <osgi.export>org.apache.helix.provisioning.yarn.example*;version="${project.version};-noimport:=true</osgi.export>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <version>6.0.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.helix</groupId>
+ <artifactId>helix-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.helix</groupId>
+ <artifactId>helix-provisioning</artifactId>
+ <version>0.7.1-incubating-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.mail</groupId>
+ <artifactId>mail</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.jms</groupId>
+ <artifactId>jms</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jdmk</groupId>
+ <artifactId>jmxtools</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jmx</groupId>
+ <artifactId>jmxri</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ <build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>appassembler-maven-plugin</artifactId>
+ <configuration>
+ <!-- Set the target configuration directory to be used in the bin scripts -->
+ <!-- <configurationDirectory>conf</configurationDirectory> -->
+ <!-- Copy the contents from "/src/main/config" to the target configuration
+ directory in the assembled application -->
+ <!-- <copyConfigurationDirectory>true</copyConfigurationDirectory> -->
+ <!-- Include the target configuration directory in the beginning of
+ the classpath declaration in the bin scripts -->
+ <includeConfigurationDirectoryInClasspath>true</includeConfigurationDirectoryInClasspath>
+ <assembleDirectory>${project.build.directory}/${project.artifactId}-pkg</assembleDirectory>
+ <!-- Extra JVM arguments that will be included in the bin scripts -->
+ <extraJvmArguments>-Xms512m -Xmx512m</extraJvmArguments>
+ <!-- Generate bin scripts for windows and unix pr default -->
+ <platforms>
+ <platform>windows</platform>
+ <platform>unix</platform>
+ </platforms>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>assemble</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes combine.children="append">
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>appassembler-maven-plugin</artifactId>
+ <configuration>
+ <programs>
+ <program>
+ <mainClass>org.apache.helix.provisioning.yarn.Client</mainClass>
+ <name>yarn-job-launcher</name>
+ </program>
+ <program>
+ <mainClass>org.apache.helix.provisioning.yarn.AppLauncher</mainClass>
+ <name>app-launcher</name>
+ </program>
+ </programs>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptors>
+ <descriptor>src/assemble/assembly.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/recipes/helloworld-provisioning-yarn/run.sh
----------------------------------------------------------------------
diff --git a/recipes/helloworld-provisioning-yarn/run.sh b/recipes/helloworld-provisioning-yarn/run.sh
new file mode 100755
index 0000000..51d4c35
--- /dev/null
+++ b/recipes/helloworld-provisioning-yarn/run.sh
@@ -0,0 +1,6 @@
+cd ../../../../
+mvn clean install -DskipTests
+cd recipes/provisioning/yarn/helloworld/
+mvn clean package -DskipTests
+chmod +x target/helloworld-pkg/bin/app-launcher.sh
+target/helloworld-pkg/bin/app-launcher.sh org.apache.helix.provisioning.yarn.example.HelloWordAppSpecFactory /Users/kgopalak/Documents/projects/incubator-helix/recipes/provisioning/yarn/helloworld/src/main/resources/hello_world_app_spec.yaml
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/recipes/helloworld-provisioning-yarn/src/assemble/assembly.xml
----------------------------------------------------------------------
diff --git a/recipes/helloworld-provisioning-yarn/src/assemble/assembly.xml b/recipes/helloworld-provisioning-yarn/src/assemble/assembly.xml
new file mode 100644
index 0000000..c2d08a1
--- /dev/null
+++ b/recipes/helloworld-provisioning-yarn/src/assemble/assembly.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<assembly>
+ <id>pkg</id>
+ <formats>
+ <format>tar</format>
+ </formats>
+ <fileSets>
+ <fileSet>
+ <directory>${project.build.directory}/${project.artifactId}-pkg/bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <lineEnding>unix</lineEnding>
+ <fileMode>0755</fileMode>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+ <fileSet>
+ <directory>${project.build.directory}/${project.artifactId}-pkg/repo/</directory>
+ <outputDirectory>repo</outputDirectory>
+ <fileMode>0755</fileMode>
+ <directoryMode>0755</directoryMode>
+ <excludes>
+ <exclude>**/*.xml</exclude>
+ </excludes>
+ </fileSet>
+ <fileSet>
+ <directory>${project.build.directory}/${project.artifactId}-pkg/conf</directory>
+ <outputDirectory>conf</outputDirectory>
+ <lineEnding>unix</lineEnding>
+ <fileMode>0755</fileMode>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}</directory>
+ <outputDirectory>/</outputDirectory>
+ <includes>
+ <include>LICENSE</include>
+ <include>NOTICE</include>
+ <include>DISCLAIMER</include>
+ </includes>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ </fileSets>
+</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/recipes/helloworld-provisioning-yarn/src/main/config/log4j.properties
----------------------------------------------------------------------
diff --git a/recipes/helloworld-provisioning-yarn/src/main/config/log4j.properties b/recipes/helloworld-provisioning-yarn/src/main/config/log4j.properties
new file mode 100644
index 0000000..91fac03
--- /dev/null
+++ b/recipes/helloworld-provisioning-yarn/src/main/config/log4j.properties
@@ -0,0 +1,31 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+##
+
+# Set root logger level to DEBUG and its only appender to A1.
+log4j.rootLogger=DEBUG,A1
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
+
+log4j.logger.org.I0Itec=ERROR
+log4j.logger.org.apache=ERROR
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWordAppSpecFactory.java
----------------------------------------------------------------------
diff --git a/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWordAppSpecFactory.java b/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWordAppSpecFactory.java
new file mode 100644
index 0000000..03c1341
--- /dev/null
+++ b/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWordAppSpecFactory.java
@@ -0,0 +1,92 @@
+package org.apache.helix.provisioning.yarn.example;
+
+import java.io.File;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.helix.provisioning.AppConfig;
+import org.apache.helix.provisioning.ApplicationSpec;
+import org.apache.helix.provisioning.ApplicationSpecFactory;
+import org.apache.helix.provisioning.yarn.example.HelloWorldService;
+import org.apache.helix.provisioning.yarn.example.HelloworldAppSpec;
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.Yaml;
+
+public class HelloWordAppSpecFactory implements ApplicationSpecFactory {
+
+ static HelloworldAppSpec data;
+
+ static {
+ HelloworldAppSpec data = new HelloworldAppSpec();
+ AppConfig appConfig = new AppConfig();
+ appConfig.setValue("k1", "v1");
+ data.setAppConfig(appConfig);
+ data.setAppName("testApp");
+ data.setAppMasterPackageUri(
+ "/Users/kgopalak/Documents/projects/incubator-helix/helix-provisioning/helix-provisioning-0.7.1-incubating-SNAPSHOT-pkg.tar");
+ HashMap<String, Map<String, String>> serviceConfigMap = new HashMap<String, Map<String, String>>();
+ serviceConfigMap.put("HelloWorld", new HashMap<String, String>());
+ serviceConfigMap.get("HelloWorld").put("k1", "v1");
+ data.setServiceConfigMap(serviceConfigMap);
+ HashMap<String, String> serviceMainClassMap = new HashMap<String, String>();
+ serviceMainClassMap.put("HelloWorld", HelloWorldService.class.getCanonicalName());
+ data.setServiceMainClassMap(serviceMainClassMap);
+ HashMap<String, String> servicePackageURIMap = new HashMap<String, String>();
+ servicePackageURIMap
+ .put(
+ "HelloWorld",
+ "/Users/kgopalak/Documents/projects/incubator-helix/helix-provisioning/helix-provisioning-0.7.1-incubating-SNAPSHOT-pkg.tar");
+ data.setServicePackageURIMap(servicePackageURIMap);
+ data.setServices(Arrays.asList(new String[] {
+ "HelloWorld"
+ })); }
+
+ @Override
+ public ApplicationSpec fromYaml(InputStream inputstream) {
+ return (ApplicationSpec) new Yaml().load(inputstream);
+ // return data;
+ }
+
+ public static void main(String[] args) {
+ DumperOptions options = new DumperOptions();
+ options.setPrettyFlow(true);
+
+ Yaml yaml = new Yaml(options);
+ HelloworldAppSpec data = new HelloworldAppSpec();
+ AppConfig appConfig = new AppConfig();
+ appConfig.setValue("k1", "v1");
+ data.setAppConfig(appConfig);
+ data.setAppName("testApp");
+ data.setAppMasterPackageUri(
+ "/Users/kgopalak/Documents/projects/incubator-helix/helix-provisioning/helix-provisioning-0.7.1-incubating-SNAPSHOT-pkg.tar");
+ HashMap<String, Map<String, String>> serviceConfigMap = new HashMap<String, Map<String, String>>();
+ serviceConfigMap.put("HelloWorld", new HashMap<String, String>());
+ serviceConfigMap.get("HelloWorld").put("k1", "v1");
+ data.setServiceConfigMap(serviceConfigMap);
+ HashMap<String, String> serviceMainClassMap = new HashMap<String, String>();
+ serviceMainClassMap.put("HelloWorld", HelloWorldService.class.getCanonicalName());
+ data.setServiceMainClassMap(serviceMainClassMap);
+ HashMap<String, String> servicePackageURIMap = new HashMap<String, String>();
+ servicePackageURIMap
+ .put(
+ "HelloWorld",
+ "/Users/kgopalak/Documents/projects/incubator-helix/helix-provisioning/helix-provisioning-0.7.1-incubating-SNAPSHOT-pkg.tar");
+ data.setServicePackageURIMap(servicePackageURIMap);
+ data.setServices(Arrays.asList(new String[] {
+ "HelloWorld"
+ }));
+ String dump = yaml.dump(data);
+ System.out.println(dump);
+
+ InputStream resourceAsStream = ClassLoader.getSystemClassLoader().getResourceAsStream("hello_world_app_spec.yaml");
+ HelloworldAppSpec load = yaml.loadAs(resourceAsStream,HelloworldAppSpec.class);
+ String dumpnew = yaml.dump(load);
+ System.out.println(dumpnew.equals(dump));
+
+ System.out.println("==================================");
+ System.out.println(dumpnew);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWorldService.java
----------------------------------------------------------------------
diff --git a/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWorldService.java b/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWorldService.java
new file mode 100644
index 0000000..8999817
--- /dev/null
+++ b/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWorldService.java
@@ -0,0 +1,41 @@
+package org.apache.helix.provisioning.yarn.example;
+
+import org.apache.helix.HelixConnection;
+import org.apache.helix.api.accessor.ResourceAccessor;
+import org.apache.helix.api.config.UserConfig;
+import org.apache.helix.api.id.ClusterId;
+import org.apache.helix.api.id.ParticipantId;
+import org.apache.helix.api.id.ResourceId;
+import org.apache.helix.api.id.StateModelDefId;
+import org.apache.helix.manager.zk.AbstractParticipantService;
+import org.apache.helix.provisioning.yarn.example.HelloWorldStateModelFactory;
+import org.apache.log4j.Logger;
+
+public class HelloWorldService extends AbstractParticipantService {
+
+ private static Logger LOG = Logger.getLogger(AbstractParticipantService.class);
+
+ static String SERVICE_NAME = "HelloWorld";
+
+ public HelloWorldService(HelixConnection connection, ClusterId clusterId,
+ ParticipantId participantId) {
+ super(connection, clusterId, participantId);
+ }
+
+ /**
+ * init method to setup appropriate call back handlers.
+ */
+ @Override
+ public void init() {
+ ClusterId clusterId = getClusterId();
+ ResourceAccessor resourceAccessor = getConnection().createResourceAccessor(clusterId);
+ UserConfig serviceConfig = resourceAccessor.readUserConfig(ResourceId.from(SERVICE_NAME));
+ LOG.info("Starting service:" + SERVICE_NAME + " with configuration:" + serviceConfig);
+
+ HelloWorldStateModelFactory stateModelFactory = new HelloWorldStateModelFactory();
+ getParticipant().getStateMachineEngine().registerStateModelFactory(
+ StateModelDefId.from("StatelessService"), stateModelFactory);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWorldStateModel.java
----------------------------------------------------------------------
diff --git a/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWorldStateModel.java b/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWorldStateModel.java
new file mode 100644
index 0000000..078d847
--- /dev/null
+++ b/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWorldStateModel.java
@@ -0,0 +1,33 @@
+package org.apache.helix.provisioning.yarn.example;
+
+import org.apache.helix.NotificationContext;
+import org.apache.helix.api.id.PartitionId;
+import org.apache.helix.model.Message;
+import org.apache.helix.participant.statemachine.StateModel;
+import org.apache.helix.participant.statemachine.StateModelInfo;
+import org.apache.helix.participant.statemachine.Transition;
+import org.apache.log4j.Logger;
+
+@StateModelInfo(initialState = "OFFLINE", states = {
+ "OFFLINE", "ONLINE", "ERROR"
+})
+public class HelloWorldStateModel extends StateModel {
+
+ private static Logger LOG = Logger.getLogger(HelloWorldStateModel.class);
+
+ public HelloWorldStateModel(PartitionId partitionId) {
+ // ignore the partitionId
+ }
+
+ @Transition(to = "ONLINE", from = "OFFLINE")
+ public void onBecomeOnlineFromOffline(Message message, NotificationContext context)
+ throws Exception {
+ LOG.info("Started HelloWorld service");
+ }
+
+ @Transition(to = "OFFLINE", from = "ONLINE")
+ public void onBecomeOfflineFromOnline(Message message, NotificationContext context)
+ throws InterruptedException {
+ LOG.info("Stopped HelloWorld service");
+ }
+}
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWorldStateModelFactory.java
----------------------------------------------------------------------
diff --git a/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWorldStateModelFactory.java b/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWorldStateModelFactory.java
new file mode 100644
index 0000000..2766f6d
--- /dev/null
+++ b/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloWorldStateModelFactory.java
@@ -0,0 +1,13 @@
+package org.apache.helix.provisioning.yarn.example;
+
+import org.apache.helix.api.id.PartitionId;
+import org.apache.helix.participant.statemachine.HelixStateModelFactory;
+import org.apache.helix.participant.statemachine.StateModel;
+import org.apache.helix.provisioning.yarn.example.HelloWorldStateModel;
+
+public class HelloWorldStateModelFactory extends HelixStateModelFactory<StateModel> {
+ @Override
+ public StateModel createNewStateModel(PartitionId partitionId) {
+ return new HelloWorldStateModel(partitionId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloworldAppSpec.java
----------------------------------------------------------------------
diff --git a/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloworldAppSpec.java b/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloworldAppSpec.java
new file mode 100644
index 0000000..588c84c
--- /dev/null
+++ b/recipes/helloworld-provisioning-yarn/src/main/java/org/apache/helix/provisioning/yarn/example/HelloworldAppSpec.java
@@ -0,0 +1,138 @@
+package org.apache.helix.provisioning.yarn.example;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.helix.api.Scope;
+import org.apache.helix.api.config.ParticipantConfig;
+import org.apache.helix.api.config.ResourceConfig;
+import org.apache.helix.api.config.ResourceConfig.Builder;
+import org.apache.helix.api.config.UserConfig;
+import org.apache.helix.api.id.ParticipantId;
+import org.apache.helix.api.id.ResourceId;
+import org.apache.helix.provisioning.AppConfig;
+import org.apache.helix.provisioning.ApplicationSpec;
+import org.apache.helix.provisioning.ServiceConfig;
+import org.apache.helix.provisioning.TaskConfig;
+
+public class HelloworldAppSpec implements ApplicationSpec {
+
+ public String _appName;
+
+ public AppConfig _appConfig;
+
+ public List<String> _services;
+
+ private String _appMasterPackageUri;
+
+ private Map<String, String> _servicePackageURIMap;
+
+ private Map<String, String> _serviceMainClassMap;
+
+ private Map<String, Map<String, String>> _serviceConfigMap;
+
+ private List<TaskConfig> _taskConfigs;
+
+ public AppConfig getAppConfig() {
+ return _appConfig;
+ }
+
+ public void setAppConfig(AppConfig appConfig) {
+ _appConfig = appConfig;
+ }
+
+ public String getAppMasterPackageUri() {
+ return _appMasterPackageUri;
+ }
+
+ public void setAppMasterPackageUri(String appMasterPackageUri) {
+ _appMasterPackageUri = appMasterPackageUri;
+ }
+
+ public Map<String, String> getServicePackageURIMap() {
+ return _servicePackageURIMap;
+ }
+
+ public void setServicePackageURIMap(Map<String, String> servicePackageURIMap) {
+ _servicePackageURIMap = servicePackageURIMap;
+ }
+
+ public Map<String, String> getServiceMainClassMap() {
+ return _serviceMainClassMap;
+ }
+
+ public void setServiceMainClassMap(Map<String, String> serviceMainClassMap) {
+ _serviceMainClassMap = serviceMainClassMap;
+ }
+
+ public Map<String, Map<String, String>> getServiceConfigMap() {
+ return _serviceConfigMap;
+ }
+
+ public void setServiceConfigMap(Map<String, Map<String, String>> serviceConfigMap) {
+ _serviceConfigMap = serviceConfigMap;
+ }
+
+ public void setAppName(String appName) {
+ _appName = appName;
+ }
+
+ public void setServices(List<String> services) {
+ _services = services;
+ }
+
+ public void setTaskConfigs(List<TaskConfig> taskConfigs) {
+ _taskConfigs = taskConfigs;
+ }
+
+ @Override
+ public String getAppName() {
+ return _appName;
+ }
+
+ @Override
+ public AppConfig getConfig() {
+ return _appConfig;
+ }
+
+ @Override
+ public List<String> getServices() {
+ return _services;
+ }
+
+ @Override
+ public URI getAppMasterPackage() {
+ try {
+ return new URI(_appMasterPackageUri);
+ } catch (URISyntaxException e) {
+ return null;
+ }
+ }
+
+ @Override
+ public URI getServicePackage(String serviceName) {
+ try {
+ return new URI(_servicePackageURIMap.get(serviceName));
+ } catch (URISyntaxException e) {
+ return null;
+ }
+ }
+
+ @Override
+ public String getServiceMainClass(String service) {
+ return _serviceMainClassMap.get(service);
+ }
+
+ @Override
+ public ServiceConfig getServiceConfig(String serviceName) {
+ return new ServiceConfig(Scope.resource(ResourceId.from(serviceName)));
+ }
+
+ @Override
+ public List<TaskConfig> getTaskConfigs() {
+ return _taskConfigs;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/recipes/helloworld-provisioning-yarn/src/main/resources/hello_world_app_spec.yaml
----------------------------------------------------------------------
diff --git a/recipes/helloworld-provisioning-yarn/src/main/resources/hello_world_app_spec.yaml b/recipes/helloworld-provisioning-yarn/src/main/resources/hello_world_app_spec.yaml
new file mode 100644
index 0000000..d8d1dd2
--- /dev/null
+++ b/recipes/helloworld-provisioning-yarn/src/main/resources/hello_world_app_spec.yaml
@@ -0,0 +1,24 @@
+!!org.apache.helix.provisioning.yarn.example.HelloworldAppSpec
+appConfig:
+ config: {
+ k1: v1
+ }
+appMasterPackageUri: 'file:///Users/kgopalak/Documents/projects/incubator-helix/recipes/provisioning/yarn/helloworld/target/helloworld-0.7.1-incubating-SNAPSHOT-pkg.tar'
+appName: testApp
+serviceConfigMap:
+ HelloWorld: {
+ num_containers: 3,
+ memory: 1024
+ }
+serviceMainClassMap: {
+ HelloWorld: org.apache.helix.provisioning.yarn.example.HelloWorldService
+}
+servicePackageURIMap: {
+ HelloWorld: 'file:///Users/kgopalak/Documents/projects/incubator-helix/recipes/provisioning/yarn/helloworld/target/helloworld-0.7.1-incubating-SNAPSHOT-pkg.tar'
+}
+services: [
+ HelloWorld]
+taskConfigs: null
+
+
+
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/recipes/helloworld-provisioning-yarn/src/test/conf/testng.xml
----------------------------------------------------------------------
diff --git a/recipes/helloworld-provisioning-yarn/src/test/conf/testng.xml b/recipes/helloworld-provisioning-yarn/src/test/conf/testng.xml
new file mode 100644
index 0000000..37bccf3
--- /dev/null
+++ b/recipes/helloworld-provisioning-yarn/src/test/conf/testng.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd">
+<suite name="Suite" parallel="none">
+ <test name="Test" preserve-order="false">
+ <packages>
+ <package name="org.apache.helix.agent"/>
+ </packages>
+ </test>
+</suite>
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/recipes/pom.xml
----------------------------------------------------------------------
diff --git a/recipes/pom.xml b/recipes/pom.xml
index 7d9952a..5d137c2 100644
--- a/recipes/pom.xml
+++ b/recipes/pom.xml
@@ -36,7 +36,7 @@ under the License.
<module>user-defined-rebalancer</module>
<module>task-execution</module>
<module>service-discovery</module>
- <module>provisioning</module>
+ <module>helloworld-provisioning-yarn</module>
</modules>
<build>
http://git-wip-us.apache.org/repos/asf/helix/blob/8992aa5a/recipes/provisioning/pom.xml
----------------------------------------------------------------------
diff --git a/recipes/provisioning/pom.xml b/recipes/provisioning/pom.xml
deleted file mode 100644
index dc5277b..0000000
--- a/recipes/provisioning/pom.xml
+++ /dev/null
@@ -1,50 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.helix.recipes</groupId>
- <artifactId>recipes</artifactId>
- <version>0.7.1-incubating-SNAPSHOT</version>
- </parent>
- <groupId>org.apache.helix.recipes.provisioning</groupId>
- <artifactId>provisioning</artifactId>
- <packaging>pom</packaging>
- <name>Apache Helix :: Recipes :: Provisioning</name>
-
- <modules>
- <module>yarn</module>
- </modules>
-
- <build>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-deploy-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
- </build>
-
-</project>