You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/09/25 23:39:02 UTC
[63/86] [abbrv] hadoop git commit: YARN-7091. Rename application to
service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
new file mode 100644
index 0000000..684f655
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+import org.apache.hadoop.yarn.service.api.records.ReadinessCheck;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Formatter;
+import java.util.Locale;
+
+/**
+ * Various utils to work with the monitor
+ */
+public final class MonitorUtils {
+ protected static final Logger LOG = LoggerFactory.getLogger(MonitorUtils
+ .class);
+
+ private MonitorUtils() {
+ }
+
+ public static String toPlural(int val) {
+ return val != 1 ? "s" : "";
+ }
+
+ /**
+ * Convert milliseconds to human time -the exact format is unspecified
+ * @param milliseconds a time in milliseconds
+ * @return a time that is converted to human intervals
+ */
+ public static String millisToHumanTime(long milliseconds) {
+ StringBuilder sb = new StringBuilder();
+ // Send all output to the Appendable object sb
+ Formatter formatter = new Formatter(sb, Locale.US);
+
+ long s = Math.abs(milliseconds / 1000);
+ long m = Math.abs(milliseconds % 1000);
+ if (milliseconds > 0) {
+ formatter.format("%d.%03ds", s, m);
+ } else if (milliseconds == 0) {
+ formatter.format("0");
+ } else {
+ formatter.format("-%d.%03ds", s, m);
+ }
+ return sb.toString();
+ }
+
+ public static Probe getProbe(ReadinessCheck readinessCheck) {
+ if (readinessCheck == null) {
+ return null;
+ }
+ if (readinessCheck.getType() == null) {
+ return null;
+ }
+ try {
+ switch (readinessCheck.getType()) {
+ case HTTP:
+ return HttpProbe.create(readinessCheck.getProps());
+ case PORT:
+ return PortProbe.create(readinessCheck.getProps());
+ default:
+ return null;
+ }
+ } catch (Throwable t) {
+ throw new IllegalArgumentException("Error creating readiness check " +
+ t);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
new file mode 100644
index 0000000..aba5859
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.util.Map;
+
+/**
+ * Probe for a port being open.
+ */
+public class PortProbe extends Probe {
+ protected static final Logger log = LoggerFactory.getLogger(PortProbe.class);
+ private final int port;
+ private final int timeout;
+
+ public PortProbe(int port, int timeout) {
+ super("Port probe of " + port + " for " + timeout + "ms", null);
+ this.port = port;
+ this.timeout = timeout;
+ }
+
+ public static PortProbe create(Map<String, String> props)
+ throws IOException {
+ int port = getPropertyInt(props, PORT_PROBE_PORT, null);
+
+ if (port >= 65536) {
+ throw new IOException(PORT_PROBE_PORT + " " + port + " is out of " +
+ "range");
+ }
+
+ int timeout = getPropertyInt(props, PORT_PROBE_CONNECT_TIMEOUT,
+ PORT_PROBE_CONNECT_TIMEOUT_DEFAULT);
+
+ return new PortProbe(port, timeout);
+ }
+
+ /**
+ * Try to connect to the (host,port); a failure to connect within
+ * the specified timeout is a failure.
+ * @param instance role instance
+ * @return the outcome
+ */
+ @Override
+ public ProbeStatus ping(ComponentInstance instance) {
+ ProbeStatus status = new ProbeStatus();
+
+ if (instance.getContainerStatus() == null || SliderUtils
+ .isEmpty(instance.getContainerStatus().getIPs())) {
+ status.fail(this, new IOException(
+ instance.getCompInstanceName() + ": IP is not available yet"));
+ return status;
+ }
+
+ String ip = instance.getContainerStatus().getIPs().get(0);
+ InetSocketAddress sockAddr = new InetSocketAddress(ip, port);
+ Socket socket = new Socket();
+ try {
+ if (log.isDebugEnabled()) {
+ log.debug(instance.getCompInstanceName() + ": Connecting " + sockAddr
+ .toString() + ", timeout=" + MonitorUtils
+ .millisToHumanTime(timeout));
+ }
+ socket.connect(sockAddr, timeout);
+ status.succeed(this);
+ } catch (Throwable e) {
+ String error =
+ instance.getCompInstanceName() + ": Probe " + sockAddr + " failed";
+ log.debug(error, e);
+ status.fail(this, new IOException(error, e));
+ } finally {
+ IOUtils.closeSocket(socket);
+ }
+ return status;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/Probe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/Probe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/Probe.java
new file mode 100644
index 0000000..3237a2b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/Probe.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Base class of all probes.
+ */
+public abstract class Probe implements MonitorKeys {
+
+ protected final Configuration conf;
+ private String name;
+
+ /**
+ * Create a probe of a specific name
+ *
+ * @param name probe name
+ * @param conf configuration being stored.
+ */
+ public Probe(String name, Configuration conf) {
+ this.name = name;
+ this.conf = conf;
+ }
+
+
+ protected void setName(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+
+ @Override
+ public String toString() {
+ return getName();
+ }
+
+ public static String getProperty(Map<String, String> props, String name,
+ String defaultValue) throws IOException {
+ String value = props.get(name);
+ if (StringUtils.isEmpty(value)) {
+ if (defaultValue == null) {
+ throw new IOException(name + " not specified");
+ }
+ return defaultValue;
+ }
+ return value;
+ }
+
+ public static int getPropertyInt(Map<String, String> props, String name,
+ Integer defaultValue) throws IOException {
+ String value = props.get(name);
+ if (StringUtils.isEmpty(value)) {
+ if (defaultValue == null) {
+ throw new IOException(name + " not specified");
+ }
+ return defaultValue;
+ }
+ return Integer.parseInt(value);
+ }
+
+ /**
+ * perform any prelaunch initialization
+ */
+ public void init() throws IOException {
+
+ }
+
+ /**
+ * Ping the endpoint. All exceptions must be caught and included in the
+ * (failure) status.
+ *
+ * @param instance instance to ping
+ * @return the status
+ */
+ public abstract ProbeStatus ping(ComponentInstance instance);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/ProbeStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/ProbeStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/ProbeStatus.java
new file mode 100644
index 0000000..bc62dcd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/ProbeStatus.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+import java.io.Serializable;
+import java.util.Date;
+
+/**
+ * Status message of a probe. This is designed to be sent over the wire, though the exception
+ * Had better be unserializable at the far end if that is to work.
+ */
+public final class ProbeStatus implements Serializable {
+ private static final long serialVersionUID = 165468L;
+
+ private long timestamp;
+ private String timestampText;
+ private boolean success;
+ private boolean realOutcome;
+ private String message;
+ private Throwable thrown;
+ private transient Probe originator;
+
+ public ProbeStatus() {
+ }
+
+ public ProbeStatus(long timestamp, String message, Throwable thrown) {
+ this.success = false;
+ this.message = message;
+ this.thrown = thrown;
+ setTimestamp(timestamp);
+ }
+
+ public ProbeStatus(long timestamp, String message) {
+ this.success = true;
+ setTimestamp(timestamp);
+ this.message = message;
+ this.thrown = null;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public void setTimestamp(long timestamp) {
+ this.timestamp = timestamp;
+ timestampText = new Date(timestamp).toString();
+ }
+
+ public boolean isSuccess() {
+ return success;
+ }
+
+ /**
+ * Set both the success and the real outcome bits to the same value
+ * @param success the new value
+ */
+ public void setSuccess(boolean success) {
+ this.success = success;
+ realOutcome = success;
+ }
+
+ public String getTimestampText() {
+ return timestampText;
+ }
+
+ public boolean getRealOutcome() {
+ return realOutcome;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+
+ public Throwable getThrown() {
+ return thrown;
+ }
+
+ public void setThrown(Throwable thrown) {
+ this.thrown = thrown;
+ }
+
+ /**
+ * Get the probe that generated this result. May be null
+ * @return a possibly null reference to a probe
+ */
+ public Probe getOriginator() {
+ return originator;
+ }
+
+ /**
+ * The probe has succeeded -capture the current timestamp, set
+ * success to true, and record any other data needed.
+ * @param probe probe
+ */
+ public void succeed(Probe probe) {
+ finish(probe, true, probe.getName(), null);
+ }
+
+ /**
+ * A probe has failed either because the test returned false, or an exception
+ * was thrown. The {@link #success} field is set to false, any exception
+ * thrown is recorded.
+ * @param probe probe that failed
+ * @param thrown an exception that was thrown.
+ */
+ public void fail(Probe probe, Throwable thrown) {
+ finish(probe, false, "Failure in " + probe, thrown);
+ }
+
+ public void finish(Probe probe, boolean succeeded, String text, Throwable thrown) {
+ setTimestamp(System.currentTimeMillis());
+ setSuccess(succeeded);
+ originator = probe;
+ message = text;
+ this.thrown = thrown;
+ }
+
+ @Override
+ public String toString() {
+ LogEntryBuilder builder = new LogEntryBuilder("Probe Status");
+ builder.elt("time", timestampText)
+ .elt("outcome", (success ? "success" : "failure"));
+
+ if (success != realOutcome) {
+ builder.elt("originaloutcome", (realOutcome ? "success" : "failure"));
+ }
+ builder.elt("message", message);
+ if (thrown != null) {
+ builder.elt("exception", thrown);
+ }
+
+ return builder.toString();
+ }
+
+ /**
+ * Flip the success bit on while the real outcome bit is kept false
+ */
+ public void markAsSuccessful() {
+ success = true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
new file mode 100644
index 0000000..0d11be2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public abstract class AbstractClientProvider {
+
+ public AbstractClientProvider() {
+ }
+
+ /**
+ * Generates a fixed format of application tags given one or more of
+ * application name, version and description. This allows subsequent query for
+ * an application with a name only, version only or description only or any
+ * combination of those as filters.
+ *
+ * @param appName name of the application
+ * @param appVersion version of the application
+ * @param appDescription brief description of the application
+ * @return
+ */
+ public static final Set<String> createApplicationTags(String appName,
+ String appVersion, String appDescription) {
+ Set<String> tags = new HashSet<>();
+ tags.add(SliderUtils.createNameTag(appName));
+ if (appVersion != null) {
+ tags.add(SliderUtils.createVersionTag(appVersion));
+ }
+ if (appDescription != null) {
+ tags.add(SliderUtils.createDescriptionTag(appDescription));
+ }
+ return tags;
+ }
+
+ /**
+ * Validate the artifact.
+ * @param artifact
+ */
+ public abstract void validateArtifact(Artifact artifact, FileSystem
+ fileSystem) throws IOException;
+
+ protected abstract void validateConfigFile(ConfigFile configFile, FileSystem
+ fileSystem) throws IOException;
+
+ /**
+ * Validate the config files.
+ * @param configFiles config file list
+ * @param fs file system
+ */
+ public void validateConfigFiles(List<ConfigFile> configFiles,
+ FileSystem fs) throws IOException {
+ Set<String> destFileSet = new HashSet<>();
+
+ for (ConfigFile file : configFiles) {
+ if (file.getType() == null) {
+ throw new IllegalArgumentException("File type is empty");
+ }
+
+ if (file.getType().equals(ConfigFile.TypeEnum.TEMPLATE) && StringUtils
+ .isEmpty(file.getSrcFile())) {
+ throw new IllegalArgumentException(
+ "Src_file is empty for " + ConfigFile.TypeEnum.TEMPLATE);
+
+ }
+ if (!StringUtils.isEmpty(file.getSrcFile())) {
+ Path p = new Path(file.getSrcFile());
+ if (!fs.exists(p)) {
+ throw new IllegalArgumentException(
+ "Src_file does not exist for config file: " + file
+ .getSrcFile());
+ }
+ }
+
+ if (StringUtils.isEmpty(file.getDestFile())) {
+ throw new IllegalArgumentException("Dest_file is empty.");
+ }
+
+ if (destFileSet.contains(file.getDestFile())) {
+ throw new IllegalArgumentException(
+ "Duplicated ConfigFile exists: " + file.getDestFile());
+ }
+ destFileSet.add(file.getDestFile());
+
+ java.nio.file.Path destPath = Paths.get(file.getDestFile());
+ if (!destPath.isAbsolute() && destPath.getNameCount() > 1) {
+ throw new IllegalArgumentException("Non-absolute dest_file has more " +
+ "than one path element");
+ }
+
+ // provider-specific validation
+ validateConfigFile(file, fs);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
new file mode 100644
index 0000000..8d607ab
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+import org.apache.hadoop.yarn.service.containerlaunch.CommandLineBuilder;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_RETRY_INTERVAL;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_RETRY_MAX;
+import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$;
+
+public abstract class AbstractProviderService implements ProviderService,
+ YarnServiceConstants {
+
+ protected static final Logger log =
+ LoggerFactory.getLogger(AbstractProviderService.class);
+
+ public abstract void processArtifact(AbstractLauncher launcher,
+ ComponentInstance compInstance, SliderFileSystem fileSystem,
+ Service service)
+ throws IOException;
+
+ public void buildContainerLaunchContext(AbstractLauncher launcher,
+ Service service, ComponentInstance instance,
+ SliderFileSystem fileSystem, Configuration yarnConf)
+ throws IOException, SliderException {
+ Component component = instance.getComponent().getComponentSpec();;
+ processArtifact(launcher, instance, fileSystem, service);
+
+ ServiceContext context =
+ instance.getComponent().getScheduler().getContext();
+ // Generate tokens (key-value pair) for config substitution.
+ // Get pre-defined tokens
+ Map<String, String> globalTokens =
+ instance.getComponent().getScheduler().globalTokens;
+ Map<String, String> tokensForSubstitution = ProviderUtils
+ .initCompTokensForSubstitute(instance);
+ tokensForSubstitution.putAll(globalTokens);
+ // Set the environment variables in launcher
+ launcher.putEnv(SliderUtils
+ .buildEnvMap(component.getConfiguration(), tokensForSubstitution));
+ launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$());
+ launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+ if (System.getenv(HADOOP_USER_NAME) != null) {
+ launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME));
+ }
+ launcher.setEnv("LANG", "en_US.UTF-8");
+ launcher.setEnv("LC_ALL", "en_US.UTF-8");
+ launcher.setEnv("LANGUAGE", "en_US.UTF-8");
+
+ for (Entry<String, String> entry : launcher.getEnv().entrySet()) {
+ tokensForSubstitution.put($(entry.getKey()), entry.getValue());
+ }
+ //TODO add component host tokens?
+// ProviderUtils.addComponentHostTokens(tokensForSubstitution, amState);
+
+ // create config file on hdfs and add local resource
+ ProviderUtils.createConfigFileAndAddLocalResource(launcher, fileSystem,
+ component, tokensForSubstitution, instance, context);
+
+ // substitute launch command
+ String launchCommand = ProviderUtils
+ .substituteStrWithTokens(component.getLaunchCommand(),
+ tokensForSubstitution);
+ CommandLineBuilder operation = new CommandLineBuilder();
+ operation.add(launchCommand);
+ operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
+ launcher.addCommand(operation.build());
+
+ // By default retry forever every 30 seconds
+ launcher.setRetryContext(YarnServiceConf
+ .getInt(CONTAINER_RETRY_MAX, -1, service.getConfiguration(),
+ yarnConf), YarnServiceConf
+ .getInt(CONTAINER_RETRY_INTERVAL, 30000, service.getConfiguration(),
+ yarnConf));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
new file mode 100644
index 0000000..0f949e0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderFactory;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.provider.docker.DockerProviderFactory;
+import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base class for factories.
+ */
+public abstract class ProviderFactory {
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(ProviderFactory.class);
+
+ protected ProviderFactory() {}
+
+ public abstract AbstractClientProvider createClientProvider();
+
+ public abstract ProviderService createServerProvider();
+
+ public static synchronized ProviderService getProviderService(Artifact
+ artifact) {
+ return createServiceProviderFactory(artifact).createServerProvider();
+ }
+
+ public static synchronized AbstractClientProvider getClientProvider(Artifact
+ artifact) {
+ return createServiceProviderFactory(artifact).createClientProvider();
+ }
+
+ /**
+ * Create a provider for a specific service
+ * @param artifact artifact
+ * @return provider factory
+ */
+ public static synchronized ProviderFactory createServiceProviderFactory(
+ Artifact artifact) {
+ if (artifact == null || artifact.getType() == null) {
+ LOG.debug("Loading service provider type default");
+ return DefaultProviderFactory.getInstance();
+ }
+ LOG.debug("Loading service provider type {}", artifact.getType());
+ switch (artifact.getType()) {
+ // TODO add handling for custom types?
+ // TODO handle service
+ case DOCKER:
+ return DockerProviderFactory.getInstance();
+ case TARBALL:
+ return TarballProviderFactory.getInstance();
+ default:
+ throw new IllegalArgumentException(String.format("Resolution error, " +
+ "%s should not be passed to createServiceProviderFactory",
+ artifact.getType()));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
new file mode 100644
index 0000000..eb721b4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+
+import java.io.IOException;
+
+public interface ProviderService {
+
+ /**
+ * Set up the entire container launch context
+ */
+ void buildContainerLaunchContext(AbstractLauncher containerLauncher,
+ Service service, ComponentInstance instance,
+ SliderFileSystem sliderFileSystem, Configuration yarnConf)
+ throws IOException, SliderException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
new file mode 100644
index 0000000..ec0c2ca
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.provider;
+
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
+import org.apache.hadoop.yarn.service.api.records.Configuration;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.utils.PublishedConfiguration;
+import org.apache.hadoop.yarn.service.utils.PublishedConfigurationOutputter;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.regex.Pattern;
+
+import static org.apache.hadoop.yarn.service.api.ServiceApiConstants.*;
+
+/**
+ * This is a factoring out of methods handy for providers. It's bonded to a log
+ * at construction time.
+ */
+public class ProviderUtils implements YarnServiceConstants {
+
+ protected static final Logger log =
+ LoggerFactory.getLogger(ProviderUtils.class);
+
+
+ /**
+ * Add oneself to the classpath. This does not work
+ * on minicluster test runs where the JAR is not built up.
+ * @param providerResources map of provider resources to add these entries to
+ * @param providerClass provider to add
+ * @param jarName name of the jar to use
+ * @param sliderFileSystem target filesystem
+ * @param tempPath path in the cluster FS for temp files
+ * @param libdir relative directory to place resources
+ * @param miniClusterTestRun true if minicluster is being used
+ * @return true if the class was found in a JAR
+ *
+ * @throws FileNotFoundException if the JAR was not found and this is NOT
+ * a mini cluster test run
+ * @throws IOException IO problems
+ * @throws SliderException any Slider problem
+ */
+ public static boolean addProviderJar(
+ Map<String, LocalResource> providerResources,
+ Class providerClass,
+ String jarName,
+ SliderFileSystem sliderFileSystem,
+ Path tempPath,
+ String libdir,
+ boolean miniClusterTestRun) throws
+ IOException,
+ SliderException {
+ try {
+ SliderUtils.putJar(providerResources,
+ sliderFileSystem,
+ providerClass,
+ tempPath,
+ libdir,
+ jarName);
+ return true;
+ } catch (FileNotFoundException e) {
+ if (miniClusterTestRun) {
+ return false;
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ /**
+ * Loads all dependency jars from the default path.
+ * @param providerResources map of provider resources to add these entries to
+ * @param sliderFileSystem target filesystem
+ * @param tempPath path in the cluster FS for temp files
+ * @param libDir relative directory to place resources
+ * @param libLocalSrcDir explicitly supplied local libs dir
+ * @throws IOException trouble copying to HDFS
+ * @throws SliderException trouble copying to HDFS
+ */
+ public static void addAllDependencyJars(
+ Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ Path tempPath,
+ String libDir,
+ String libLocalSrcDir)
+ throws IOException, SliderException {
+ if (SliderUtils.isSet(libLocalSrcDir)) {
+ File file = new File(libLocalSrcDir);
+ if (!file.exists() || !file.isDirectory()) {
+ throw new BadCommandArgumentsException(
+ "Supplied lib src dir %s is not valid", libLocalSrcDir);
+ }
+ }
+ SliderUtils.putAllJars(providerResources, sliderFileSystem, tempPath,
+ libDir, libLocalSrcDir);
+ }
+
+ public static String substituteStrWithTokens(String content,
+ Map<String, String> tokensForSubstitution) {
+ for (Map.Entry<String, String> token : tokensForSubstitution.entrySet()) {
+ content =
+ content.replaceAll(Pattern.quote(token.getKey()), token.getValue());
+ }
+ return content;
+ }
+
+ // configs will be substituted by corresponding env in tokenMap
+ public static void substituteMapWithTokens(Map<String, String> configs,
+ Map<String, String> tokenMap) {
+ for (Map.Entry<String, String> entry : configs.entrySet()) {
+ String value = entry.getValue();
+ if (tokenMap != null) {
+ for (Map.Entry<String, String> token : tokenMap.entrySet()) {
+ value =
+ value.replaceAll(Pattern.quote(token.getKey()), token.getValue());
+ }
+ }
+ entry.setValue(value);
+ }
+ }
+
+ /**
+ * Localize the service keytabs for the service.
+ * @param launcher container launcher
+ * @param fileSystem file system
+ * @throws IOException trouble uploading to HDFS
+ */
+ public void localizeServiceKeytabs(AbstractLauncher launcher,
+ SliderFileSystem fileSystem, Service service) throws IOException {
+
+ Configuration conf = service.getConfiguration();
+ String keytabPathOnHost =
+ conf.getProperty(YarnServiceConf.KEY_AM_KEYTAB_LOCAL_PATH);
+ if (SliderUtils.isUnset(keytabPathOnHost)) {
+ String amKeytabName =
+ conf.getProperty(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_NAME);
+ String keytabDir =
+ conf.getProperty(YarnServiceConf.KEY_HDFS_KEYTAB_DIR);
+ // we need to localize the keytab files in the directory
+ Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null,
+ service.getName());
+ boolean serviceKeytabsDeployed = false;
+ if (fileSystem.getFileSystem().exists(keytabDirPath)) {
+ FileStatus[] keytabs = fileSystem.getFileSystem().listStatus(
+ keytabDirPath);
+ LocalResource keytabRes;
+ for (FileStatus keytab : keytabs) {
+ if (!amKeytabName.equals(keytab.getPath().getName())
+ && keytab.getPath().getName().endsWith(".keytab")) {
+ serviceKeytabsDeployed = true;
+ log.info("Localizing keytab {}", keytab.getPath().getName());
+ keytabRes = fileSystem.createAmResource(keytab.getPath(),
+ LocalResourceType.FILE);
+ launcher.addLocalResource(KEYTAB_DIR + "/" +
+ keytab.getPath().getName(),
+ keytabRes);
+ }
+ }
+ }
+ if (!serviceKeytabsDeployed) {
+ log.warn("No service keytabs for the service have been localized. "
+ + "If the service requires keytabs for secure operation, "
+ + "please ensure that the required keytabs have been uploaded "
+ + "to the folder {}", keytabDirPath);
+ }
+ }
+ }
+
+ // 1. Create all config files for a component on hdfs for localization
+ // 2. Add the config file to localResource
+ public static synchronized void createConfigFileAndAddLocalResource(
+ AbstractLauncher launcher, SliderFileSystem fs, Component component,
+ Map<String, String> tokensForSubstitution, ComponentInstance instance,
+ ServiceContext context) throws IOException {
+ Path compDir =
+ new Path(new Path(fs.getAppDir(), "components"), component.getName());
+ Path compInstanceDir =
+ new Path(compDir, instance.getCompInstanceName());
+ if (!fs.getFileSystem().exists(compInstanceDir)) {
+ log.info(instance.getCompInstanceId() + ": Creating dir on hdfs: " + compInstanceDir);
+ fs.getFileSystem().mkdirs(compInstanceDir,
+ new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
+ instance.setCompInstanceDir(compInstanceDir);
+ } else {
+ log.info("Component instance conf dir already exists: " + compInstanceDir);
+ }
+
+ if (log.isDebugEnabled()) {
+ log.debug("Tokens substitution for component instance: " + instance
+ .getCompInstanceName() + System.lineSeparator()
+ + tokensForSubstitution);
+ }
+
+ for (ConfigFile originalFile : component.getConfiguration().getFiles()) {
+ ConfigFile configFile = originalFile.copy();
+ String fileName = new Path(configFile.getDestFile()).getName();
+
+ // substitute file name
+ for (Map.Entry<String, String> token : tokensForSubstitution.entrySet()) {
+ configFile.setDestFile(configFile.getDestFile()
+ .replaceAll(Pattern.quote(token.getKey()), token.getValue()));
+ }
+
+ Path remoteFile = new Path(compInstanceDir, fileName);
+ if (!fs.getFileSystem().exists(remoteFile)) {
+ log.info("Saving config file on hdfs for component " + instance
+ .getCompInstanceName() + ": " + configFile);
+
+ if (configFile.getSrcFile() != null) {
+ // Load config file template
+ switch (configFile.getType()) {
+ case HADOOP_XML:
+ // Hadoop_xml_template
+ resolveHadoopXmlTemplateAndSaveOnHdfs(fs.getFileSystem(),
+ tokensForSubstitution, configFile, remoteFile, context);
+ break;
+ case TEMPLATE:
+ // plain-template
+ resolvePlainTemplateAndSaveOnHdfs(fs.getFileSystem(),
+ tokensForSubstitution, configFile, remoteFile, context);
+ break;
+ default:
+ log.info("Not supporting loading src_file for " + configFile);
+ break;
+ }
+ } else {
+ // non-template
+ resolveNonTemplateConfigsAndSaveOnHdfs(fs, tokensForSubstitution,
+ instance, configFile, fileName, remoteFile);
+ }
+ }
+
+ // Add resource for localization
+ LocalResource configResource =
+ fs.createAmResource(remoteFile, LocalResourceType.FILE);
+ File destFile = new File(configFile.getDestFile());
+ String symlink = APP_CONF_DIR + "/" + fileName;
+ if (destFile.isAbsolute()) {
+ launcher.addLocalResource(symlink, configResource,
+ configFile.getDestFile());
+ log.info("Add config file for localization: " + symlink + " -> "
+ + configResource.getResource().getFile() + ", dest mount path: "
+ + configFile.getDestFile());
+ } else {
+ launcher.addLocalResource(symlink, configResource);
+ log.info("Add config file for localization: " + symlink + " -> "
+ + configResource.getResource().getFile());
+ }
+ }
+ }
+
+ private static void resolveNonTemplateConfigsAndSaveOnHdfs(SliderFileSystem fs,
+ Map<String, String> tokensForSubstitution, ComponentInstance instance,
+ ConfigFile configFile, String fileName, Path remoteFile)
+ throws IOException {
+ // substitute non-template configs
+ substituteMapWithTokens(configFile.getProps(), tokensForSubstitution);
+
+ // write configs onto hdfs
+ PublishedConfiguration publishedConfiguration =
+ new PublishedConfiguration(fileName,
+ configFile.getProps().entrySet());
+ if (!fs.getFileSystem().exists(remoteFile)) {
+ PublishedConfigurationOutputter configurationOutputter =
+ PublishedConfigurationOutputter.createOutputter(
+ ConfigFormat.resolve(configFile.getType().toString()),
+ publishedConfiguration);
+ try (FSDataOutputStream os = fs.getFileSystem().create(remoteFile)) {
+ configurationOutputter.save(os);
+ os.flush();
+ }
+ } else {
+ log.info("Component instance = " + instance.getCompInstanceName()
+ + ", config file already exists: " + remoteFile);
+ }
+ }
+
+ // 1. substitute config template - only handle hadoop_xml format
+ // 2. save on hdfs
+ @SuppressWarnings("unchecked")
+ private static void resolveHadoopXmlTemplateAndSaveOnHdfs(FileSystem fs,
+ Map<String, String> tokensForSubstitution, ConfigFile configFile,
+ Path remoteFile, ServiceContext context) throws IOException {
+ Map<String, String> conf;
+ try {
+ conf = (Map<String, String>) context.configCache.get(configFile);
+ } catch (ExecutionException e) {
+ log.info("Failed to load config file: " + configFile, e);
+ return;
+ }
+ // make a copy for substitution
+ org.apache.hadoop.conf.Configuration confCopy =
+ new org.apache.hadoop.conf.Configuration(false);
+ for (Map.Entry<String, String> entry : conf.entrySet()) {
+ confCopy.set(entry.getKey(), entry.getValue());
+ }
+ // substitute properties
+ for (Map.Entry<String, String> entry : configFile.getProps().entrySet()) {
+ confCopy.set(entry.getKey(), entry.getValue());
+ }
+ // substitute env variables
+ for (Map.Entry<String, String> entry : confCopy) {
+ String val = entry.getValue();
+ if (val != null) {
+ for (Map.Entry<String, String> token : tokensForSubstitution
+ .entrySet()) {
+ val = val.replaceAll(Pattern.quote(token.getKey()), token.getValue());
+ confCopy.set(entry.getKey(), val);
+ }
+ }
+ }
+ // save on hdfs
+ try (OutputStream output = fs.create(remoteFile)) {
+ confCopy.writeXml(output);
+ log.info("Reading config from: " + configFile.getSrcFile()
+ + ", writing to: " + remoteFile);
+ }
+ }
+
+ // 1) read the template as a string
+ // 2) do token substitution
+ // 3) save on hdfs
+ private static void resolvePlainTemplateAndSaveOnHdfs(FileSystem fs,
+ Map<String, String> tokensForSubstitution, ConfigFile configFile,
+ Path remoteFile, ServiceContext context) {
+ String content;
+ try {
+ content = (String) context.configCache.get(configFile);
+ } catch (ExecutionException e) {
+ log.info("Failed to load config file: " + configFile, e);
+ return;
+ }
+ // substitute tokens
+ content = substituteStrWithTokens(content, tokensForSubstitution);
+
+ try (OutputStream output = fs.create(remoteFile)) {
+ org.apache.commons.io.IOUtils.write(content, output);
+ } catch (IOException e) {
+ log.info("Failed to create " + remoteFile);
+ }
+ }
+
+ /**
+ * Get initial component token map to be substituted into config values.
+ * @return tokens to replace
+ */
+ public static Map<String, String> initCompTokensForSubstitute(
+ ComponentInstance instance) {
+ Map<String, String> tokens = new HashMap<>();
+ tokens.put(COMPONENT_NAME, instance.getCompSpec().getName());
+ tokens
+ .put(COMPONENT_NAME_LC, instance.getCompSpec().getName().toLowerCase());
+ tokens.put(COMPONENT_INSTANCE_NAME, instance.getCompInstanceName());
+ tokens.put(CONTAINER_ID, instance.getContainer().getId().toString());
+ tokens.put(COMPONENT_ID,
+ String.valueOf(instance.getCompInstanceId().getId()));
+ return tokens;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java
new file mode 100644
index 0000000..0920a9c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultClientProvider.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.defaultImpl;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+
+public class DefaultClientProvider extends AbstractClientProvider {
+
+ public DefaultClientProvider() {
+ }
+
+ @Override
+ public void validateArtifact(Artifact artifact, FileSystem fileSystem) {
+ }
+
+ @Override
+ protected void validateConfigFile(ConfigFile configFile, FileSystem
+ fileSystem) throws IOException {
+ // validate dest_file is not absolute
+ if (Paths.get(configFile.getDestFile()).isAbsolute()) {
+ throw new IllegalArgumentException(
+ "Dest_file must not be absolute path: " + configFile.getDestFile());
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java
new file mode 100644
index 0000000..868bba8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderFactory.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.defaultImpl;
+
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.provider.ProviderService;
+import org.apache.hadoop.yarn.service.provider.ProviderFactory;
+
+public final class DefaultProviderFactory extends ProviderFactory {
+ private static final ProviderFactory FACTORY = new
+ DefaultProviderFactory();
+
+ private DefaultProviderFactory() {}
+
+ private static class Client {
+ static final AbstractClientProvider PROVIDER = new DefaultClientProvider();
+ }
+
+ private static class Server {
+ static final ProviderService PROVIDER = new DefaultProviderService();
+ }
+
+ @Override
+ public AbstractClientProvider createClientProvider() {
+ return Client.PROVIDER;
+ }
+
+ @Override
+ public ProviderService createServerProvider() {
+ return Server.PROVIDER;
+ }
+
+ public static ProviderFactory getInstance() {
+ return FACTORY;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
new file mode 100644
index 0000000..a3a0c1f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/defaultImpl/DefaultProviderService.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.defaultImpl;
+
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+
+import java.io.IOException;
+
+public class DefaultProviderService extends AbstractProviderService {
+
+ @Override
+ public void processArtifact(AbstractLauncher launcher,
+ ComponentInstance compInstance, SliderFileSystem fileSystem,
+ Service service)
+ throws IOException {
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java
new file mode 100644
index 0000000..d4a2254
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerClientProvider.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.docker;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
+
+import java.io.IOException;
+
+public class DockerClientProvider extends AbstractClientProvider
+ implements YarnServiceConstants {
+
+ public DockerClientProvider() {
+ super();
+ }
+
+ @Override
+ public void validateArtifact(Artifact artifact, FileSystem fileSystem) {
+ if (artifact == null) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
+ }
+ if (StringUtils.isEmpty(artifact.getId())) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
+ }
+ }
+
+ @Override
+ protected void validateConfigFile(ConfigFile configFile, FileSystem
+ fileSystem) throws IOException {
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java
new file mode 100644
index 0000000..f30c002
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.docker;
+
+public interface DockerKeys {
+ String PROVIDER_DOCKER = "docker";
+ String DOCKER_PREFIX = "docker.";
+ String DOCKER_IMAGE = DOCKER_PREFIX + "image";
+ String DOCKER_NETWORK = DOCKER_PREFIX + "network";
+ String DOCKER_USE_PRIVILEGED = DOCKER_PREFIX + "usePrivileged";
+ String DOCKER_START_COMMAND = DOCKER_PREFIX + "startCommand";
+
+ String DEFAULT_DOCKER_NETWORK = "bridge";
+ Boolean DEFAULT_DOCKER_USE_PRIVILEGED = false;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java
new file mode 100644
index 0000000..57330ab
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.docker;
+
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.provider.ProviderService;
+import org.apache.hadoop.yarn.service.provider.ProviderFactory;
+
+public class DockerProviderFactory extends ProviderFactory {
+ private static final ProviderFactory FACTORY = new
+ DockerProviderFactory();
+
+ private DockerProviderFactory() {
+ }
+
+ private static class Client {
+ static final AbstractClientProvider PROVIDER = new DockerClientProvider();
+ }
+
+ private static class Server {
+ static final ProviderService PROVIDER = new DockerProviderService();
+ }
+
+ @Override
+ public AbstractClientProvider createClientProvider() {
+ return Client.PROVIDER;
+ }
+
+ @Override
+ public ProviderService createServerProvider() {
+ return Server.PROVIDER;
+ }
+
+ public static ProviderFactory getInstance() {
+ return FACTORY;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
new file mode 100644
index 0000000..0741947
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.docker;
+
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+
+import java.io.IOException;
+import java.text.MessageFormat;
+
+public class DockerProviderService extends AbstractProviderService
+ implements DockerKeys {
+
+ public void processArtifact(AbstractLauncher launcher,
+ ComponentInstance compInstance, SliderFileSystem fileSystem,
+ Service service) throws IOException{
+ launcher.setYarnDockerMode(true);
+ launcher.setDockerImage(compInstance.getCompSpec().getArtifact().getId());
+ launcher.setDockerNetwork(compInstance.getCompSpec().getConfiguration()
+ .getProperty(DOCKER_NETWORK, DEFAULT_DOCKER_NETWORK));
+ String domain = compInstance.getComponent().getScheduler().getConfig()
+ .get(RegistryConstants.KEY_DNS_DOMAIN);
+ String hostname;
+ if (domain == null || domain.isEmpty()) {
+ hostname = MessageFormat
+ .format("{0}.{1}.{2}", compInstance.getCompInstanceName(),
+ service.getName(), RegistryUtils.currentUser());
+ } else {
+ hostname = MessageFormat
+ .format("{0}.{1}.{2}.{3}", compInstance.getCompInstanceName(),
+ service.getName(), RegistryUtils.currentUser(), domain);
+ }
+ launcher.setDockerHostname(hostname);
+ launcher.setRunPrivilegedContainer(
+ compInstance.getCompSpec().getRunPrivilegedContainer());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java
new file mode 100644
index 0000000..01f7b20
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballClientProvider.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.tarball;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+
+public class TarballClientProvider extends AbstractClientProvider
+ implements YarnServiceConstants {
+
+ public TarballClientProvider() {
+ }
+
+ @Override
+ public void validateArtifact(Artifact artifact, FileSystem fs)
+ throws IOException {
+ if (artifact == null) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
+ }
+ if (StringUtils.isEmpty(artifact.getId())) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
+ }
+ Path p = new Path(artifact.getId());
+ if (!fs.exists(p)) {
+ throw new IllegalArgumentException( "Artifact tarball does not exist "
+ + artifact.getId());
+ }
+ }
+
+ @Override
+ protected void validateConfigFile(ConfigFile configFile, FileSystem
+ fileSystem) throws IOException {
+ // validate dest_file is not absolute
+ if (Paths.get(configFile.getDestFile()).isAbsolute()) {
+ throw new IllegalArgumentException(
+ "Dest_file must not be absolute path: " + configFile.getDestFile());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java
new file mode 100644
index 0000000..9d81f66
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.tarball;
+
+import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
+import org.apache.hadoop.yarn.service.provider.ProviderService;
+import org.apache.hadoop.yarn.service.provider.ProviderFactory;
+
+public class TarballProviderFactory extends ProviderFactory {
+ private static final ProviderFactory FACTORY = new
+ TarballProviderFactory();
+
+ private TarballProviderFactory() {
+ }
+
+ private static class Client {
+ static final AbstractClientProvider PROVIDER = new TarballClientProvider();
+ }
+
+ private static class Server {
+ static final ProviderService PROVIDER = new TarballProviderService();
+ }
+
+ @Override
+ public AbstractClientProvider createClientProvider() {
+ return Client.PROVIDER;
+ }
+
+ @Override
+ public ProviderService createServerProvider() {
+ return Server.PROVIDER;
+ }
+
+ public static ProviderFactory getInstance() {
+ return FACTORY;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
new file mode 100644
index 0000000..9f29c8b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service.provider.tarball;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.provider.AbstractProviderService;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+
+import java.io.IOException;
+
+public class TarballProviderService extends AbstractProviderService {
+
+ @Override
+ public void processArtifact(AbstractLauncher launcher,
+ ComponentInstance instance, SliderFileSystem fileSystem,
+ Service service)
+ throws IOException {
+ Path artifact = new Path(instance.getCompSpec().getArtifact().getId());
+ if (!fileSystem.isFile(artifact)) {
+ throw new IOException(
+ "Package doesn't exist as a resource: " + artifact.toString());
+ }
+ log.info("Adding resource {}", artifact.toString());
+ LocalResourceType type = LocalResourceType.ARCHIVE;
+ LocalResource packageResource = fileSystem.createAmResource(artifact, type);
+ launcher.addLocalResource(APP_LIB_DIR, packageResource);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java
new file mode 100644
index 0000000..56634f6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/CustomRegistryConstants.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.registry;
+
+/**
+ * These are constants unique to the Slider AM
+ */
+public class CustomRegistryConstants {
+
+ public static final String MANAGEMENT_REST_API =
+ "classpath:org.apache.slider.management";
+
+ public static final String REGISTRY_REST_API =
+ "classpath:org.apache.slider.registry";
+
+ public static final String PUBLISHER_REST_API =
+ "classpath:org.apache.slider.publisher";
+
+ public static final String PUBLISHER_CONFIGURATIONS_API =
+ "classpath:org.apache.slider.publisher.configurations";
+
+ public static final String PUBLISHER_EXPORTS_API =
+ "classpath:org.apache.slider.publisher.exports";
+
+ public static final String PUBLISHER_DOCUMENTS_API =
+ "classpath:org.apache.slider.publisher.documents";
+
+ public static final String AGENT_SECURE_REST_API =
+ "classpath:org.apache.slider.agents.secure";
+
+ public static final String AGENT_ONEWAY_REST_API =
+ "classpath:org.apache.slider.agents.oneway";
+
+ public static final String AM_IPC_PROTOCOL =
+ "classpath:org.apache.slider.appmaster.ipc";
+
+ public static final String AM_REST_BASE =
+ "classpath:org.apache.slider.client.rest";
+
+ public static final String WEB_UI = "http://";
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org