You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2016/12/13 22:53:37 UTC
[45/74] [abbrv] hadoop git commit: YARN-5610. Initial code for native
services REST API. Contributed by Gour Saha
http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java
new file mode 100644
index 0000000..a3780cc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.resource;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * Resource determines the amount of resources (vcores, memory, network, etc.)
+ * usable by a container. This field determines the resource to be applied for
+ * all the containers of a component or application. The resource specified at
+ * the app (or global) level can be overriden at the component level. Only one
+ * of profile OR cpu & memory are exepected. It raises a validation
+ * exception otherwise.
+ **/
+
+@ApiModel(description = "Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or application. The resource specified at the app (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are exepected. It raises a validation exception otherwise.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+public class Resource extends BaseResource {
+ private static final long serialVersionUID = -6431667797380250037L;
+
+ private String profile = null;
+ private Integer cpus = null;
+ private String memory = null;
+
+ /**
+ * Each resource profile has a unique id which is associated with a
+ * cluster-level predefined memory, cpus, etc.
+ **/
+ public Resource profile(String profile) {
+ this.profile = profile;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc.")
+ @JsonProperty("profile")
+ public String getProfile() {
+ return profile;
+ }
+
+ public void setProfile(String profile) {
+ this.profile = profile;
+ }
+
+ /**
+ * Amount of vcores allocated to each container (optional but overrides cpus
+ * in profile if specified).
+ **/
+ public Resource cpus(Integer cpus) {
+ this.cpus = cpus;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Amount of vcores allocated to each container (optional but overrides cpus in profile if specified).")
+ @JsonProperty("cpus")
+ public Integer getCpus() {
+ return cpus;
+ }
+
+ public void setCpus(Integer cpus) {
+ this.cpus = cpus;
+ }
+
+ /**
+ * Amount of memory allocated to each container (optional but overrides memory
+ * in profile if specified). Currently accepts only an integer value and
+ * default unit is in MB.
+ **/
+ public Resource memory(String memory) {
+ this.memory = memory;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.")
+ @JsonProperty("memory")
+ public String getMemory() {
+ return memory;
+ }
+
+ public void setMemory(String memory) {
+ this.memory = memory;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Resource resource = (Resource) o;
+ return Objects.equals(this.profile, resource.profile)
+ && Objects.equals(this.cpus, resource.cpus)
+ && Objects.equals(this.memory, resource.memory);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(profile, cpus, memory);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Resource {\n");
+
+ sb.append(" profile: ").append(toIndentedString(profile)).append("\n");
+ sb.append(" cpus: ").append(toIndentedString(cpus)).append("\n");
+ sb.append(" memory: ").append(toIndentedString(memory)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ return super.clone();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java
new file mode 100644
index 0000000..4c16546
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.utils;
+
+public interface RestApiConstants {
+ String CONTEXT_ROOT = "/services/v1";
+ String APPLICATIONS_API_RESOURCE_PATH = "/applications";
+ String CONTAINERS_API_RESOURCE_PATH = "/containers";
+ String SLIDER_APPMASTER_COMPONENT_NAME = "slider-appmaster";
+ String SLIDER_CONFIG_SCHEMA = "http://example.org/specification/v2.0.0";
+ String METAINFO_SCHEMA_VERSION = "2.1";
+ String COMPONENT_TYPE_YARN_DOCKER = "yarn_docker";
+
+ String DEFAULT_START_CMD = "/bootstrap/privileged-centos6-sshd";
+ String DEFAULT_COMPONENT_NAME = "DEFAULT";
+ String DEFAULT_IMAGE = "centos:centos6";
+ String DEFAULT_NETWORK = "bridge";
+ String DEFAULT_COMMAND_PATH = "/usr/bin/docker";
+ String DEFAULT_USE_NETWORK_SCRIPT = "yes";
+
+ String PLACEHOLDER_APP_NAME = "${APP_NAME}";
+ String PLACEHOLDER_APP_COMPONENT_NAME = "${APP_COMPONENT_NAME}";
+ String PLACEHOLDER_COMPONENT_ID = "${COMPONENT_ID}";
+
+ String PROPERTY_REST_SERVICE_HOST = "REST_SERVICE_HOST";
+ String PROPERTY_REST_SERVICE_PORT = "REST_SERVICE_PORT";
+ String PROPERTY_APP_LIFETIME = "docker.lifetime";
+ String PROPERTY_APP_RUNAS_USER = "APP_RUNAS_USER";
+ Long DEFAULT_UNLIMITED_LIFETIME = -1l;
+
+ Integer HTTP_STATUS_CODE_ACCEPTED = 202;
+ String ARTIFACT_TYPE_SLIDER_ZIP = "slider-zip";
+
+ Integer GET_APPLICATIONS_THREAD_POOL_SIZE = 200;
+
+ String PROPERTY_PYTHON_PATH = "python.path";
+ String PROPERTY_COMPONENT_TYPE = "site.global.component_type";
+ String PROPERTY_DNS_DEPENDENCY = "site.global.dns.dependency";
+
+ String COMPONENT_TYPE_EXTERNAL = "external";
+
+ String COMMAND_ORDER_SUFFIX_START = "-START";
+ String COMMAND_ORDER_SUFFIX_STARTED = "-STARTED";
+ String EXPORT_GROUP_NAME = "QuickLinks";
+
+ Integer ERROR_CODE_APP_DOES_NOT_EXIST = 404001;
+ Integer ERROR_CODE_APP_IS_NOT_RUNNING = 404002;
+ Integer ERROR_CODE_APP_SUBMITTED_BUT_NOT_RUNNING_YET = 404003;
+ Integer ERROR_CODE_APP_NAME_INVALID = 404004;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java
new file mode 100644
index 0000000..685f85a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.utils;
+
+public interface RestApiErrorMessages {
+ String ERROR_APPLICATION_NAME_INVALID =
+ "Application name is either empty or not provided";
+ String ERROR_APPLICATION_NAME_INVALID_FORMAT =
+ "Application name is not valid - only lower case letters, digits,"
+ + " underscore and hyphen are allowed";
+
+ String ERROR_APPLICATION_NOT_RUNNING = "Application not running";
+ String ERROR_APPLICATION_DOES_NOT_EXIST = "Application not found";
+
+ String ERROR_SUFFIX_FOR_COMPONENT =
+ " for component %s (nor at the global level)";
+ String ERROR_ARTIFACT_INVALID = "Artifact is not provided";
+ String ERROR_ARTIFACT_FOR_COMP_INVALID =
+ ERROR_ARTIFACT_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_ARTIFACT_ID_INVALID =
+ "Artifact id (like docker image name) is either empty or not provided";
+ String ERROR_ARTIFACT_ID_FOR_COMP_INVALID =
+ ERROR_ARTIFACT_ID_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+
+ String ERROR_RESOURCE_INVALID = "Resource is not provided";
+ String ERROR_RESOURCE_FOR_COMP_INVALID =
+ ERROR_RESOURCE_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_RESOURCE_MEMORY_INVALID =
+ "Application resource or memory not provided";
+ String ERROR_RESOURCE_CPUS_INVALID =
+ "Application resource or cpus not provided";
+ String ERROR_RESOURCE_CPUS_INVALID_RANGE =
+ "Unacceptable no of cpus specified, either zero or negative";
+ String ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID =
+ ERROR_RESOURCE_MEMORY_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID =
+ ERROR_RESOURCE_CPUS_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE =
+ ERROR_RESOURCE_CPUS_INVALID_RANGE
+ + " for component %s (or at the global level)";
+ String ERROR_CONTAINERS_COUNT_INVALID =
+ "Required no of containers not specified";
+ String ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID =
+ ERROR_CONTAINERS_COUNT_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+
+ String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED =
+ "Cannot specify" + " cpus/memory along with profile";
+ String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED =
+ ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED
+ + " for component %s";
+ String ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET =
+ "Resource profile is not " + "supported yet. Please specify cpus/memory.";
+
+ String ERROR_APPLICATION_IN_USE = "Application name is already in use";
+ String ERROR_NULL_ARTIFACT_ID =
+ "Artifact Id can not be null if artifact type is none";
+ String ERROR_ABSENT_NUM_OF_INSTANCE =
+ "Num of instances should appear either globally or per component";
+ String ERROR_ABSENT_LAUNCH_COMMAND =
+ "launch command should appear if type is slider-zip or none";
+
+ String ERROR_QUICKLINKS_FOR_COMP_INVALID =
+ "Quicklinks specified at component level, needs corresponding values set at application level";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java
new file mode 100644
index 0000000..b1b6d7c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.webapp;
+
+import static org.apache.hadoop.yarn.services.utils.RestApiConstants.*;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.URI;
+import java.util.Arrays;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.mortbay.jetty.webapp.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class launches the web application using Hadoop HttpServer2 (which uses
+ * an embedded Jetty container). This is the entry point to your application.
+ * The Java command used to launch this app should call the main method.
+ */
+public class ApplicationApiWebApp extends AbstractService {
+ private static final Logger logger = LoggerFactory
+ .getLogger(ApplicationApiWebApp.class);
+ private static final String SEP = ";";
+
+ // REST API server for YARN native services
+ private HttpServer2 applicationApiServer;
+
+ public static void main(String[] args) throws IOException {
+ ApplicationApiWebApp apiWebApp = new ApplicationApiWebApp();
+ try {
+ apiWebApp.startWebApp();
+ } catch (Exception e) {
+ if (apiWebApp != null) {
+ apiWebApp.close();
+ }
+ }
+ }
+
+ public ApplicationApiWebApp() {
+ super(ApplicationApiWebApp.class.getName());
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ startWebApp();
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (applicationApiServer != null) {
+ applicationApiServer.stop();
+ }
+ super.serviceStop();
+ }
+
+ protected void startWebApp() throws IOException {
+ // The port that we should run on can be set into an environment variable
+ // Look for that variable and default to 9191 if it isn't there.
+ String webPort = System.getenv(PROPERTY_REST_SERVICE_PORT);
+ if (StringUtils.isEmpty(webPort)) {
+ webPort = "9191";
+ }
+
+ String webHost = System.getenv(PROPERTY_REST_SERVICE_HOST);
+ if (StringUtils.isEmpty(webHost)) {
+ webHost = InetAddress.getLocalHost().getHostName();
+ }
+ logger.info("YARN native services REST API running on host {} and port {}",
+ webHost, webPort);
+ logger.info("Configuration = {}", getConfig());
+
+ applicationApiServer = new HttpServer2.Builder()
+ .setName("services-rest-api")
+ .addEndpoint(URI.create("http://" + webHost + ":" + webPort)).build();
+
+ String apiPackages = "org.apache.hadoop.yarn.services.api" + SEP
+ + "org.apache.hadoop.yarn.services.api.impl" + SEP
+ + "org.apache.hadoop.yarn.services.resource" + SEP
+ + "org.apache.hadoop.yarn.services.utils" + SEP
+ + "org.apache.hadoop.yarn.services.webapp" + SEP
+ + GenericExceptionHandler.class.getPackage().getName() + SEP
+ + YarnJacksonJaxbJsonProvider.class.getPackage().getName();
+ applicationApiServer.addJerseyResourcePackage(apiPackages, CONTEXT_ROOT
+ + "/*");
+
+ try {
+ logger.info("Application starting up. Logging start...");
+ applicationApiServer.start();
+ logger.info("Server status = {}", applicationApiServer.toString());
+ for (Configuration conf : applicationApiServer.getWebAppContext()
+ .getConfigurations()) {
+ logger.info("Configurations = {}", conf);
+ }
+ logger.info("Context Path = {}", Arrays.asList(applicationApiServer
+ .getWebAppContext().getContextPath()));
+ logger.info("ResourceBase = {}", Arrays.asList(applicationApiServer
+ .getWebAppContext().getResourceBase()));
+ logger.info("War = {}",
+ Arrays.asList(applicationApiServer.getWebAppContext().getWar()));
+ } catch (Exception ex) {
+ logger.error("Hadoop HttpServer2 App **failed**", ex);
+ throw ex;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/log4j-server.properties
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/log4j-server.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/log4j-server.properties
new file mode 100644
index 0000000..8c679b9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/log4j-server.properties
@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is the log4j configuration for YARN Services REST API Server
+
+# Log rotation based on size (100KB) with a max of 10 backup files
+log4j.rootLogger=INFO, restservicelog
+log4j.threshhold=ALL
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n
+
+log4j.appender.restservicelog=org.apache.log4j.RollingFileAppender
+log4j.appender.restservicelog.layout=org.apache.log4j.PatternLayout
+log4j.appender.restservicelog.File=${REST_SERVICE_LOG_DIR}/restservice.log
+log4j.appender.restservicelog.MaxFileSize=1GB
+log4j.appender.restservicelog.MaxBackupIndex=10
+
+# log layout skips stack-trace creation operations by avoiding line numbers and method
+log4j.appender.restservicelog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+# debug edition is much more expensive
+#log4j.appender.restservicelog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+# configure stderr
+# set the conversion pattern of stderr
+# Print the date in ISO 8601 format
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.Target=System.err
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
+
+# for debugging REST API Service
+#log4j.logger.org.apache.hadoop.yarn.services=DEBUG
+
+# uncomment to debug service lifecycle issues
+#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+
+# uncomment for YARN operations
+#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+
+# uncomment this to debug security problems
+#log4j.logger.org.apache.hadoop.security=DEBUG
+
+#crank back on some noise
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdfs=WARN
+log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR
+
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
+log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.curator.framework.state=ERROR
+log4j.logger.org.apache.curator.framework.imps=WARN
+
+log4j.logger.org.mortbay.log=DEBUG
http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/services-rest-api/app
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/services-rest-api/app b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/services-rest-api/app
new file mode 100644
index 0000000..6a077b1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/services-rest-api/app
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DON'T DELETE. REST WEBAPP RUN SCRIPT WILL STOP WORKING.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh
new file mode 100644
index 0000000..9f15b7e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export SLIDER_VERSION=${project.version}
+export HDP_VERSION=${HDP_VERSION:-$SLIDER_VERSION}
+export SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
+export LIB_PARENT_DIR=`dirname $SCRIPT_DIR`
+export JAVA_HOME=${JAVA_HOME:-/usr/jdk64/jdk1.8.0_40}
+export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop/conf}
+export REST_SERVICE_PORT=${REST_SERVICE_PORT:-9191}
+export APP_RUNAS_USER=${APP_RUNAS_USER:-root}
+export REST_SERVICE_LOG_DIR=${REST_SERVICE_LOG_DIR:-/tmp/}
+export JAVA_OPTS="-Xms256m -Xmx1024m -XX:+PrintGC -Xloggc:$REST_SERVICE_LOG_DIR/gc.log"
+$JAVA_HOME/bin/java $JAVA_OPTS -cp .:$HADOOP_CONF_DIR:$LIB_PARENT_DIR/services-api/*:$LIB_PARENT_DIR/slider/* -DREST_SERVICE_LOG_DIR=$REST_SERVICE_LOG_DIR -Dlog4j.configuration=log4j-server.properties -Dslider.libdir=$LIB_PARENT_DIR/slider org.apache.hadoop.yarn.services.webapp.ApplicationApiWebApp 1>>$REST_SERVICE_LOG_DIR/restservice-out.log 2>&1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml
new file mode 100644
index 0000000..f2f8b5b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<web-app xmlns="http://java.sun.com/xml/ns/javaee" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd"
+ version="3.0">
+
+ <servlet>
+ <servlet-name>Jersey REST API</servlet-name>
+ <servlet-class>com.sun.jersey.spi.container.servlet.ServletContainer</servlet-class>
+ <init-param>
+ <param-name>com.sun.jersey.config.property.packages</param-name>
+ <param-value>org.apache.hadoop.yarn.services.webapp,org.apache.hadoop.yarn.services.api,org.apache.hadoop.yarn.services.resource,org.apache.hadoop.yarn.services.api.impl</param-value>
+ </init-param>
+ <init-param>
+ <param-name>com.sun.jersey.api.json.POJOMappingFeature</param-name>
+ <param-value>true</param-value>
+ </init-param>
+ <load-on-startup>1</load-on-startup>
+ </servlet>
+ <servlet-mapping>
+ <servlet-name>Jersey REST API</servlet-name>
+ <url-pattern>/*</url-pattern>
+ </servlet-mapping>
+</web-app>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java
new file mode 100644
index 0000000..a03ab69
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.api.impl;
+
+import static org.apache.hadoop.yarn.services.utils.RestApiConstants.*;
+import static org.apache.hadoop.yarn.services.utils.RestApiErrorMessages.*;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.services.resource.Application;
+import org.apache.hadoop.yarn.services.resource.Artifact;
+import org.apache.hadoop.yarn.services.resource.Resource;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test class for application life time monitor feature test.
+ */
+@RunWith(PowerMockRunner.class)
+@SuppressStaticInitializationFor("org.apache.hadoop.yarn.services.api.impl.ApplicationApiService")
+public class TestApplicationApiService {
+ private static final Logger logger = LoggerFactory
+ .getLogger(TestApplicationApiService.class);
+ private static String EXCEPTION_PREFIX = "Should have thrown exception: ";
+ private static String NO_EXCEPTION_PREFIX = "Should not have thrown exception: ";
+ private ApplicationApiService appApiService;
+
+ @Before
+ public void setup() throws Exception {
+ appApiService = new ApplicationApiService();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ }
+
+ @Test(timeout = 90000)
+ public void testValidateApplicationPostPayload() throws Exception {
+ Application app = new Application();
+ Map<String, String> compNameArtifactIdMap = new HashMap<>();
+
+ // no name
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ Assert.fail(EXCEPTION_PREFIX + "application with no name");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage());
+ }
+
+ // bad format name
+ String[] badNames = { "4finance", "Finance", "finance@home" };
+ for (String badName : badNames) {
+ app.setName(badName);
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ Assert.fail(EXCEPTION_PREFIX + "application with bad name " + badName);
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID_FORMAT,
+ e.getMessage());
+ }
+ }
+
+ // no artifact
+ app.setName("finance_home");
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ Assert.fail(EXCEPTION_PREFIX + "application with no artifact");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_ARTIFACT_INVALID, e.getMessage());
+ }
+
+ // no artifact id
+ Artifact artifact = new Artifact();
+ app.setArtifact(artifact);
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
+ }
+
+ // if artifact is of type APPLICATION then everything is valid here
+ artifact.setType(Artifact.TypeEnum.APPLICATION);
+ artifact.setId("app.io/hbase:facebook_0.2");
+ app.setNumberOfContainers(5l);
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ } catch (IllegalArgumentException e) {
+ logger.error("application attributes specified should be valid here", e);
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ // default-component, default-lifetime and the property component_type
+ // should get assigned here
+ Assert.assertEquals(app.getComponents().get(0).getName(),
+ DEFAULT_COMPONENT_NAME);
+ Assert.assertEquals(app.getLifetime(), DEFAULT_UNLIMITED_LIFETIME);
+ Assert.assertEquals("Property not set",
+ app.getConfiguration().getProperties().get(PROPERTY_COMPONENT_TYPE),
+ COMPONENT_TYPE_EXTERNAL);
+
+ // unset artifact type, default component and no of containers to test other
+ // validation logic
+ artifact.setType(null);
+ app.setComponents(null);
+ app.setNumberOfContainers(null);
+
+ // resource not specified
+ artifact.setId("docker.io/centos:centos7");
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ Assert.fail(EXCEPTION_PREFIX + "application with no resource");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_RESOURCE_INVALID, e.getMessage());
+ }
+
+ // memory not specified
+ Resource res = new Resource();
+ app.setResource(res);
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ Assert.fail(EXCEPTION_PREFIX + "application with no memory");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_RESOURCE_MEMORY_INVALID, e.getMessage());
+ }
+
+ // cpus not specified
+ res.setMemory("2gb");
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ Assert.fail(EXCEPTION_PREFIX + "application with no cpu");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_RESOURCE_CPUS_INVALID, e.getMessage());
+ }
+
+ // invalid no of cpus
+ res.setCpus(-2);
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ Assert.fail(
+ EXCEPTION_PREFIX + "application with invalid no of cpups");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_RESOURCE_CPUS_INVALID_RANGE, e.getMessage());
+ }
+
+ // number of containers not specified
+ res.setCpus(2);
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ Assert.fail(
+ EXCEPTION_PREFIX + "application with no container count");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_CONTAINERS_COUNT_INVALID, e.getMessage());
+ }
+
+ // specifying profile along with cpus/memory raises exception
+ res.setProfile("hbase_finance_large");
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ Assert.fail(EXCEPTION_PREFIX
+ + "application with resource profile along with cpus/memory");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED,
+ e.getMessage());
+ }
+
+ // currently resource profile alone is not supported.
+ // TODO: remove the next test once it is supported.
+ res.setCpus(null);
+ res.setMemory(null);
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ Assert.fail(EXCEPTION_PREFIX
+ + "application with resource profile only - NOT SUPPORTED");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET,
+ e.getMessage());
+ }
+
+ // unset profile here and add cpus/memory back
+ res.setProfile(null);
+ res.setCpus(2);
+ res.setMemory("2gb");
+
+ // everything valid here
+ app.setNumberOfContainers(5l);
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ } catch (IllegalArgumentException e) {
+ logger.error("application attributes specified should be valid here", e);
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ // Now test with components
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
index 493e03a..2cec8bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
@@ -37,7 +37,7 @@
<module>hadoop-yarn-applications-distributedshell</module>
<module>hadoop-yarn-applications-unmanaged-am-launcher</module>
<module>hadoop-yarn-slider</module>
-
+ <module>hadoop-yarn-services-api</module>
</modules>
<profiles>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org