You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2016/12/07 21:10:03 UTC
[03/76] [abbrv] hadoop git commit: YARN-5461. Initial code ported
from slider-core module. (jianhe)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
new file mode 100644
index 0000000..e436ae9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
@@ -0,0 +1,495 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.services.security;
+
+import com.google.inject.Singleton;
+import org.apache.commons.io.FileUtils;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.nio.charset.Charset;
+import java.text.MessageFormat;
+
+@Singleton
+public class CertificateManager {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(CertificateManager.class);
+
+ private static final String GEN_SRVR_KEY = "openssl genrsa -des3 " +
+ "-passout pass:{0} -out {1}" + File.separator + "{2} 4096 ";
+ private static final String GEN_SRVR_REQ = "openssl req -passin pass:{0} " +
+ "-new -key {1}" + File.separator + "{2} -out {1}" + File.separator +
+ "{5} -config {1}" + File.separator + "ca.config " +
+ "-subj {6} -batch";
+ private static final String SIGN_SRVR_CRT = "openssl ca -create_serial " +
+ "-out {1}" + File.separator + "{3} -days 365 -keyfile {1}" + File.separator
+ + "{2} -key {0} -selfsign -extensions jdk7_ca -config {1}" + File.separator
+ + "ca.config -batch -infiles {1}" + File.separator + "{5}";
+ private static final String EXPRT_KSTR = "openssl pkcs12 -export" +
+ " -in {2}" + File.separator + "{4} -inkey {2}" + File.separator +
+ "{3} -certfile {2}" + File.separator + "{4} -out {2}" + File.separator +
+ "{5} -password pass:{1} -passin pass:{0} \n";
+ private static final String REVOKE_AGENT_CRT = "openssl ca " +
+ "-config {0}" + File.separator + "ca.config -keyfile {0}" +
+ File.separator + "{4} -revoke {0}" + File.separator + "{2} -batch " +
+ "-passin pass:{3} -cert {0}" + File.separator + "{5}";
+ private static final String SIGN_AGENT_CRT = "openssl ca -config " +
+ "{0}" + File.separator + "ca.config -in {0}" + File.separator +
+ "{1} -out {0}" + File.separator + "{2} -batch -passin pass:{3} " +
+ "-keyfile {0}" + File.separator + "{4} -cert {0}" + File.separator + "{5}";
+ private static final String GEN_AGENT_KEY="openssl req -new -newkey " +
+ "rsa:1024 -nodes -keyout {0}" + File.separator +
+ "{2}.key -subj {1} -out {0}" + File.separator + "{2}.csr " +
+ "-config {3}" + File.separator + "ca.config ";
+ private String passphrase;
+ private String applicationName;
+
+
+ public void initialize(MapOperations compOperations) throws SliderException {
+ String hostname = null;
+ try {
+ hostname = InetAddress.getLocalHost().getCanonicalHostName();
+ } catch (UnknownHostException e) {
+ hostname = "localhost";
+ }
+ this.initialize(compOperations, hostname, null, null);
+ }
+
+ /**
+ * Verify that root certificate exists, generate it otherwise.
+ */
+ public void initialize(MapOperations compOperations,
+ String hostname, String containerId,
+ String appName) throws SliderException {
+ SecurityUtils.initializeSecurityParameters(compOperations);
+
+ LOG.info("Initialization of root certificate");
+ boolean certExists = isCertExists();
+ LOG.info("Certificate exists:" + certExists);
+
+ this.applicationName = appName;
+
+ if (!certExists) {
+ generateAMKeystore(hostname, containerId);
+ }
+
+ }
+
+ /**
+ * Checks root certificate state.
+ * @return "true" if certificate exists
+ */
+ private boolean isCertExists() {
+
+ String srvrKstrDir = SecurityUtils.getSecurityDir();
+ String srvrCrtName = SliderKeys.CRT_FILE_NAME;
+ File certFile = new File(srvrKstrDir + File.separator + srvrCrtName);
+ LOG.debug("srvrKstrDir = " + srvrKstrDir);
+ LOG.debug("srvrCrtName = " + srvrCrtName);
+ LOG.debug("certFile = " + certFile.getAbsolutePath());
+
+ return certFile.exists();
+ }
+
+ public void setPassphrase(String passphrase) {
+ this.passphrase = passphrase;
+ }
+
+ class StreamConsumer extends Thread
+ {
+ InputStream is;
+ boolean logOutput;
+
+ StreamConsumer(InputStream is, boolean logOutput)
+ {
+ this.is = is;
+ this.logOutput = logOutput;
+ }
+
+ StreamConsumer(InputStream is)
+ {
+ this(is, false);
+ }
+
+ public void run()
+ {
+ try
+ {
+ InputStreamReader isr = new InputStreamReader(is,
+ Charset.forName("UTF8"));
+ BufferedReader br = new BufferedReader(isr);
+ String line;
+ while ( (line = br.readLine()) != null)
+ if (logOutput) {
+ LOG.info(line);
+ }
+ } catch (IOException e)
+ {
+ LOG.error("Error during processing of process stream", e);
+ }
+ }
+ }
+
+
+ /**
+ * Runs os command
+ *
+ * @return command execution exit code
+ */
+ private int runCommand(String command) throws SliderException {
+ int exitCode = -1;
+ String line = null;
+ Process process = null;
+ BufferedReader br= null;
+ try {
+ process = Runtime.getRuntime().exec(command);
+ StreamConsumer outputConsumer =
+ new StreamConsumer(process.getInputStream(), true);
+ StreamConsumer errorConsumer =
+ new StreamConsumer(process.getErrorStream(), true);
+
+ outputConsumer.start();
+ errorConsumer.start();
+
+ try {
+ process.waitFor();
+ SecurityUtils.logOpenSslExitCode(command, process.exitValue());
+ exitCode = process.exitValue();
+ if (exitCode != 0) {
+ throw new SliderException(exitCode, "Error running command %s", command);
+ }
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ if (br != null) {
+ try {
+ br.close();
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
+ }
+ }
+ }
+
+ return exitCode;//some exception occurred
+
+ }
+
+ public synchronized void generateContainerCertificate(String hostname,
+ String identifier) {
+ LOG.info("Generation of certificate for {}", hostname);
+
+ String srvrKstrDir = SecurityUtils.getSecurityDir();
+ Object[] scriptArgs = {srvrKstrDir, getSubjectDN(hostname, identifier,
+ this.applicationName), identifier, SecurityUtils.getSecurityDir()};
+
+ try {
+ String command = MessageFormat.format(GEN_AGENT_KEY, scriptArgs);
+ runCommand(command);
+
+ signAgentCertificate(identifier);
+
+ } catch (SliderException e) {
+ LOG.error("Error generating the agent certificate", e);
+ }
+ }
+
+ public synchronized SecurityStore generateContainerKeystore(String hostname,
+ String requesterId,
+ String role,
+ String keystorePass)
+ throws SliderException {
+ LOG.info("Generation of container keystore for container {} on {}",
+ requesterId, hostname);
+
+ generateContainerCertificate(hostname, requesterId);
+
+ // come up with correct args to invoke keystore command
+ String srvrCrtPass = SecurityUtils.getKeystorePass();
+ String srvrKstrDir = SecurityUtils.getSecurityDir();
+ String containerCrtName = requesterId + ".crt";
+ String containerKeyName = requesterId + ".key";
+ String kstrName = getKeystoreFileName(requesterId, role);
+
+ Object[] scriptArgs = {srvrCrtPass, keystorePass, srvrKstrDir,
+ containerKeyName, containerCrtName, kstrName};
+
+ String command = MessageFormat.format(EXPRT_KSTR, scriptArgs);
+ runCommand(command);
+
+ return new SecurityStore(new File(srvrKstrDir, kstrName),
+ SecurityStore.StoreType.keystore);
+ }
+
+ private static String getKeystoreFileName(String containerId,
+ String role) {
+ return String.format("keystore-%s-%s.p12", containerId,
+ role != null ? role : "");
+ }
+
+ private void generateAMKeystore(String hostname, String containerId)
+ throws SliderException {
+ LOG.info("Generation of server certificate");
+
+ String srvrKstrDir = SecurityUtils.getSecurityDir();
+ String srvrCrtName = SliderKeys.CRT_FILE_NAME;
+ String srvrCsrName = SliderKeys.CSR_FILE_NAME;
+ String srvrKeyName = SliderKeys.KEY_FILE_NAME;
+ String kstrName = SliderKeys.KEYSTORE_FILE_NAME;
+ String srvrCrtPass = SecurityUtils.getKeystorePass();
+
+ Object[] scriptArgs = {srvrCrtPass, srvrKstrDir, srvrKeyName,
+ srvrCrtName, kstrName, srvrCsrName, getSubjectDN(hostname, containerId,
+ this.applicationName)};
+
+ String command = MessageFormat.format(GEN_SRVR_KEY, scriptArgs);
+ runCommand(command);
+
+ command = MessageFormat.format(GEN_SRVR_REQ, scriptArgs);
+ runCommand(command);
+
+ command = MessageFormat.format(SIGN_SRVR_CRT, scriptArgs);
+ runCommand(command);
+
+ Object[] keystoreArgs = {srvrCrtPass, srvrCrtPass, srvrKstrDir, srvrKeyName,
+ srvrCrtName, kstrName, srvrCsrName};
+ command = MessageFormat.format(EXPRT_KSTR, keystoreArgs);
+ runCommand(command);
+ }
+
+ public SecurityStore generateContainerTruststore(String containerId,
+ String role,
+ String truststorePass)
+ throws SliderException {
+
+ String srvrKstrDir = SecurityUtils.getSecurityDir();
+ String srvrCrtName = SliderKeys.CRT_FILE_NAME;
+ String srvrCsrName = SliderKeys.CSR_FILE_NAME;
+ String srvrKeyName = SliderKeys.KEY_FILE_NAME;
+ String kstrName = getTruststoreFileName(role, containerId);
+ String srvrCrtPass = SecurityUtils.getKeystorePass();
+
+ Object[] scriptArgs = {srvrCrtPass, truststorePass, srvrKstrDir, srvrKeyName,
+ srvrCrtName, kstrName, srvrCsrName};
+
+ String command = MessageFormat.format(EXPRT_KSTR, scriptArgs);
+ runCommand(command);
+
+ return new SecurityStore(new File(srvrKstrDir, kstrName),
+ SecurityStore.StoreType.truststore);
+ }
+
+ private static String getTruststoreFileName(String role, String containerId) {
+ return String.format("truststore-%s-%s.p12", containerId,
+ role != null ? role : "");
+ }
+
+ /**
+ * Returns server certificate content
+ * @return string with server certificate content
+ */
+ public String getServerCert() {
+ File certFile = getServerCertficateFilePath();
+ String srvrCrtContent = null;
+ try {
+ srvrCrtContent = FileUtils.readFileToString(certFile);
+ } catch (IOException e) {
+ LOG.error(e.getMessage());
+ }
+ return srvrCrtContent;
+ }
+
+ public static File getServerCertficateFilePath() {
+ return new File(String.format("%s%s%s",
+ SecurityUtils.getSecurityDir(),
+ File.separator,
+ SliderKeys.CRT_FILE_NAME));
+ }
+
+ public static File getAgentCertficateFilePath(String containerId) {
+ return new File(String.format("%s%s%s.crt",
+ SecurityUtils.getSecurityDir(),
+ File.separator,
+ containerId));
+ }
+
+ public static File getContainerKeystoreFilePath(String containerId,
+ String role) {
+ return new File(SecurityUtils.getSecurityDir(), getKeystoreFileName(
+ containerId,
+ role
+ ));
+ }
+
+ public static File getContainerTruststoreFilePath(String role,
+ String containerId) {
+ return new File(SecurityUtils.getSecurityDir(),
+ getTruststoreFileName(role, containerId));
+ }
+
+ public static File getAgentKeyFilePath(String containerId) {
+ return new File(String.format("%s%s%s.key",
+ SecurityUtils.getSecurityDir(),
+ File.separator,
+ containerId));
+ }
+
+ /**
+ * Signs agent certificate
+ * Adds agent certificate to server keystore
+ * @return string with agent signed certificate content
+ */
+ public synchronized SignCertResponse signAgentCrt(String agentHostname,
+ String agentCrtReqContent,
+ String passphraseAgent) {
+ SignCertResponse response = new SignCertResponse();
+ LOG.info("Signing of agent certificate");
+ LOG.info("Verifying passphrase");
+
+ if (!this.passphrase.equals(passphraseAgent.trim())) {
+ LOG.warn("Incorrect passphrase from the agent");
+ response.setResult(SignCertResponse.ERROR_STATUS);
+ response.setMessage("Incorrect passphrase from the agent");
+ return response;
+ }
+
+ String srvrKstrDir = SecurityUtils.getSecurityDir();
+ String srvrCrtPass = SecurityUtils.getKeystorePass();
+ String srvrCrtName = SliderKeys.CRT_FILE_NAME;
+ String srvrKeyName = SliderKeys.KEY_FILE_NAME;
+ String agentCrtReqName = agentHostname + ".csr";
+ String agentCrtName = agentHostname + ".crt";
+
+ Object[] scriptArgs = {srvrKstrDir, agentCrtReqName, agentCrtName,
+ srvrCrtPass, srvrKeyName, srvrCrtName};
+
+ //Revoke previous agent certificate if exists
+ File agentCrtFile = new File(srvrKstrDir + File.separator + agentCrtName);
+
+ String command = null;
+ if (agentCrtFile.exists()) {
+ LOG.info("Revoking of " + agentHostname + " certificate.");
+ command = MessageFormat.format(REVOKE_AGENT_CRT, scriptArgs);
+ try {
+ runCommand(command);
+ } catch (SliderException e) {
+ int commandExitCode = e.getExitCode();
+ response.setResult(SignCertResponse.ERROR_STATUS);
+ response.setMessage(
+ SecurityUtils.getOpenSslCommandResult(command, commandExitCode));
+ return response;
+ }
+ }
+
+ File agentCrtReqFile = new File(srvrKstrDir + File.separator +
+ agentCrtReqName);
+ try {
+ FileUtils.writeStringToFile(agentCrtReqFile, agentCrtReqContent);
+ } catch (IOException e1) {
+ // TODO Auto-generated catch block
+ e1.printStackTrace();
+ }
+
+ command = MessageFormat.format(SIGN_AGENT_CRT, scriptArgs);
+
+ LOG.debug(SecurityUtils.hideOpenSslPassword(command));
+ try {
+ runCommand(command);
+ } catch (SliderException e) {
+ int commandExitCode = e.getExitCode();
+ response.setResult(SignCertResponse.ERROR_STATUS);
+ response.setMessage(
+ SecurityUtils.getOpenSslCommandResult(command, commandExitCode));
+ return response;
+ }
+
+ String agentCrtContent = "";
+ try {
+ agentCrtContent = FileUtils.readFileToString(agentCrtFile);
+ } catch (IOException e) {
+ e.printStackTrace();
+ LOG.error("Error reading signed agent certificate");
+ response.setResult(SignCertResponse.ERROR_STATUS);
+ response.setMessage("Error reading signed agent certificate");
+ return response;
+ }
+ response.setResult(SignCertResponse.OK_STATUS);
+ response.setSignedCa(agentCrtContent);
+ //LOG.info(ShellCommandUtil.getOpenSslCommandResult(command, commandExitCode));
+ return response;
+ }
+
+ private String signAgentCertificate (String containerId)
+ throws SliderException {
+ String srvrKstrDir = SecurityUtils.getSecurityDir();
+ String srvrCrtPass = SecurityUtils.getKeystorePass();
+ String srvrCrtName = SliderKeys.CRT_FILE_NAME;
+ String srvrKeyName = SliderKeys.KEY_FILE_NAME;
+ String agentCrtReqName = containerId + ".csr";
+ String agentCrtName = containerId + ".crt";
+
+ // server certificate must exist already
+ if (!(new File(srvrKstrDir, srvrCrtName).exists())) {
+ throw new SliderException("CA certificate not generated");
+ }
+
+ Object[] scriptArgs = {srvrKstrDir, agentCrtReqName, agentCrtName,
+ srvrCrtPass, srvrKeyName, srvrCrtName};
+
+ //Revoke previous agent certificate if exists
+ File agentCrtFile = new File(srvrKstrDir + File.separator + agentCrtName);
+
+ String command;
+ if (agentCrtFile.exists()) {
+ LOG.info("Revoking of " + containerId + " certificate.");
+ command = MessageFormat.format(REVOKE_AGENT_CRT, scriptArgs);
+ runCommand(command);
+ }
+
+ command = MessageFormat.format(SIGN_AGENT_CRT, scriptArgs);
+
+ LOG.debug(SecurityUtils.hideOpenSslPassword(command));
+ runCommand(command);
+
+ return agentCrtName;
+
+ }
+
+ private String getSubjectDN(String hostname, String containerId,
+ String appName) {
+ return String.format("/CN=%s%s%s",
+ hostname,
+ containerId != null ? "/OU=" + containerId : "",
+ appName != null ? "/OU=" + appName : "");
+
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/KeystoreGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/KeystoreGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/KeystoreGenerator.java
new file mode 100644
index 0000000..e2339d5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/KeystoreGenerator.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.services.security;
+
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.SliderException;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ *
+ */
+public class KeystoreGenerator extends AbstractSecurityStoreGenerator {
+
+
+ public KeystoreGenerator(CertificateManager certificateMgr) {
+ super(certificateMgr);
+ }
+
+ @Override
+ public SecurityStore generate(String hostname, String containerId,
+ AggregateConf instanceDefinition,
+ MapOperations compOps, String role)
+ throws SliderException, IOException {
+ SecurityStore keystore = null;
+ String password = getStorePassword(
+ instanceDefinition.getAppConf().credentials, compOps, role);
+ if (password != null) {
+ keystore =
+ certificateMgr.generateContainerKeystore(hostname, containerId, role,
+ password);
+ }
+ return keystore;
+ }
+
+ @Override
+ String getPassword(MapOperations compOps) {
+ return compOps.get(
+ compOps.get(SliderKeys.COMP_KEYSTORE_PASSWORD_PROPERTY_KEY));
+ }
+
+ @Override
+ String getAlias(MapOperations compOps) {
+ return compOps.getOption(SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_KEY,
+ SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_DEFAULT);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStore.java
new file mode 100644
index 0000000..fc54267
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStore.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.services.security;
+
+import java.io.File;
+
+/**
+ *
+ */
+public class SecurityStore {
+ private File file;
+
+ public enum StoreType {truststore, keystore}
+
+ private StoreType type;
+
+ public String getType() {
+ return type.name();
+ }
+
+ public File getFile() {
+ return file;
+ }
+
+ public SecurityStore(File file,
+ StoreType type) {
+
+ this.file = file;
+ this.type = type;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ SecurityStore that = (SecurityStore) o;
+
+ if (file != null ? !file.equals(that.file) : that.file != null)
+ return false;
+ if (type != that.type) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = file != null ? file.hashCode() : 0;
+ result = 31 * result + (type != null ? type.hashCode() : 0);
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStoreGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStoreGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStoreGenerator.java
new file mode 100644
index 0000000..a814988
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStoreGenerator.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.services.security;
+
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.SliderException;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ *
+ */
+public interface SecurityStoreGenerator {
+
+ SecurityStore generate(String hostname,
+ String containerId,
+ AggregateConf instanceDefinition,
+ MapOperations compOps,
+ String role)
+ throws SliderException, IOException;
+
+ boolean isStoreRequested(MapOperations compOps);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
new file mode 100644
index 0000000..e82ad84
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.services.security;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.core.conf.MapOperations;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+//import java.nio.file.Files;
+//import java.nio.file.Path;
+//import java.nio.file.Paths;
+//import java.nio.file.attribute.PosixFilePermission;
+//import java.nio.file.attribute.PosixFilePermissions;
+
+
+/**
+ *
+ */
+public class SecurityUtils {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SecurityUtils.class);
+
+ private static String CA_CONFIG_CONTENTS = "HOME = .\n"
+ + "RANDFILE = $ENV::HOME/.rnd\n\n"
+ + "[ ca ]\n"
+ + "default_ca = CA_CLIENT\n"
+ + "[ CA_CLIENT ]\n"
+ + "dir = ${SEC_DIR}/db\n"
+ + "certs = $dir/certs\n"
+ + "new_certs_dir = $dir/newcerts\n"
+ + "\n"
+ + "database = $dir/index.txt\n"
+ + "serial = $dir/serial\n"
+ + "default_days = 365 \n"
+ + "\n"
+ + "default_crl_days = 7 \n"
+ + "default_md = sha256 \n"
+ + "\n"
+ + "policy = policy_anything \n"
+ + "\n"
+ + "[ policy_anything ]\n"
+ + "countryName = optional\n"
+ + "stateOrProvinceName = optional\n"
+ + "localityName = optional\n"
+ + "organizationName = optional\n"
+ + "organizationalUnitName = optional\n"
+ + "commonName = optional\n"
+ + "emailAddress = optional\n"
+ + "\n"
+ + "[req]\n"
+ + "distinguished_name = req_distinguished_name\n"
+ + "\n"
+ + "[ req_distinguished_name ]\n"
+ + "\n"
+ + "[ jdk7_ca ]\n"
+ + "subjectKeyIdentifier = hash\n"
+ + "authorityKeyIdentifier = keyid:always,issuer:always\n"
+ + "basicConstraints = CA:true\n";
+
+ private static final String PASS_TOKEN = "pass:";
+ private static String keystorePass;
+ private static String securityDir;
+
+ public static void logOpenSslExitCode(String command, int exitCode) {
+ if (exitCode == 0) {
+ LOG.info(getOpenSslCommandResult(command, exitCode));
+ } else {
+ LOG.warn(getOpenSslCommandResult(command, exitCode));
+ }
+
+ }
+
+ public static String hideOpenSslPassword(String command){
+ int start = command.indexOf(PASS_TOKEN);
+ while (start >= 0) {
+ start += PASS_TOKEN.length();
+ CharSequence cs = command.subSequence(start, command.indexOf(" ", start));
+ command = command.replace(cs, "****");
+ start = command.indexOf(PASS_TOKEN, start + 1);
+ }
+ return command;
+ }
+
+ public static String getOpenSslCommandResult(String command, int exitCode) {
+ return new StringBuilder().append("Command ")
+ .append(hideOpenSslPassword(command))
+ .append(" was finished with exit code: ")
+ .append(exitCode).append(" - ")
+ .append(getOpenSslExitCodeDescription(exitCode)).toString();
+ }
+
+ private static String getOpenSslExitCodeDescription(int exitCode) {
+ switch (exitCode) {
+ case 0: {
+ return "the operation was completed successfully.";
+ }
+ case 1: {
+ return "an error occurred parsing the command options.";
+ }
+ case 2: {
+ return "one of the input files could not be read.";
+ }
+ case 3: {
+ return "an error occurred creating the PKCS#7 file or when reading the MIME message.";
+ }
+ case 4: {
+ return "an error occurred decrypting or verifying the message.";
+ }
+ case 5: {
+ return "the message was verified correctly but an error occurred writing out the signers certificates.";
+ }
+ default:
+ return "unsupported code";
+ }
+ }
+
+ public static void writeCaConfigFile(String path) throws IOException {
+ String contents = CA_CONFIG_CONTENTS.replace("${SEC_DIR}", path);
+ FileUtils.writeStringToFile(new File(path, "ca.config"), contents);
+ }
+
+ public static String getKeystorePass() {
+ return keystorePass;
+ }
+
+ public static String getSecurityDir() {
+ return securityDir;
+ }
+
+ public static void initializeSecurityParameters(MapOperations configMap) {
+ initializeSecurityParameters(configMap, false);
+ }
+
+ public static void initializeSecurityParameters(MapOperations configMap,
+ boolean persistPassword) {
+ String keyStoreLocation = configMap.getOption(
+ SliderXmlConfKeys.KEY_KEYSTORE_LOCATION, getDefaultKeystoreLocation());
+ if (keyStoreLocation == null) {
+ LOG.error(SliderXmlConfKeys.KEY_KEYSTORE_LOCATION
+ + " is not specified. Unable to initialize security params.");
+ return;
+ }
+ File secDirFile = new File(keyStoreLocation).getParentFile();
+ if (!secDirFile.exists()) {
+ // create entire required directory structure
+ File dbDir = new File(secDirFile, "db");
+ File newCertsDir = new File(dbDir, "newcerts");
+ newCertsDir.mkdirs();
+ RawLocalFileSystem fileSystem = null;
+ try {
+ fileSystem = new RawLocalFileSystem();
+ FsPermission permissions = new FsPermission(FsAction.ALL, FsAction.NONE,
+ FsAction.NONE);
+ fileSystem.setPermission(new Path(dbDir.getAbsolutePath()),
+ permissions);
+ fileSystem.setPermission(new Path(dbDir.getAbsolutePath()), permissions);
+ fileSystem.setPermission(new Path(newCertsDir.getAbsolutePath()),
+ permissions);
+ File indexFile = new File(dbDir, "index.txt");
+ indexFile.createNewFile();
+ SecurityUtils.writeCaConfigFile(secDirFile.getAbsolutePath().replace('\\', '/'));
+
+ } catch (IOException e) {
+ LOG.error("Unable to create SSL configuration directories/files", e);
+ } finally {
+ if (fileSystem != null) {
+ try {
+ fileSystem.close();
+ } catch (IOException e) {
+ LOG.warn("Unable to close fileSystem", e);
+ }
+ }
+ }
+ // need to create the password
+ }
+ keystorePass = getKeystorePassword(secDirFile, persistPassword);
+ securityDir = secDirFile.getAbsolutePath();
+ }
+
+ private static String getKeystorePassword(File secDirFile,
+ boolean persistPassword) {
+ File passFile = new File(secDirFile, SliderKeys.CRT_PASS_FILE_NAME);
+ String password = null;
+ if (!passFile.exists()) {
+ LOG.info("Generating keystore password");
+ password = RandomStringUtils.randomAlphanumeric(
+ Integer.valueOf(SliderKeys.PASS_LEN));
+ if (persistPassword) {
+ try {
+ FileUtils.writeStringToFile(passFile, password);
+ passFile.setWritable(true);
+ passFile.setReadable(true);
+ } catch (IOException e) {
+ e.printStackTrace();
+ throw new RuntimeException(
+ "Error creating certificate password file");
+ }
+ }
+ } else {
+ LOG.info("Reading password from existing file");
+ try {
+ password = FileUtils.readFileToString(passFile);
+ password = password.replaceAll("\\p{Cntrl}", "");
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ return password;
+ }
+
+ private static String getDefaultKeystoreLocation() {
+ File workDir = null;
+ try {
+ workDir = new File(FileUtils.getTempDirectory().getAbsolutePath()
+ + "/sec" + System.currentTimeMillis());
+ if (!workDir.mkdirs()) {
+ throw new IOException("Unable to create temporary security directory");
+ }
+ } catch (IOException e) {
+ LOG.warn("Unable to create security directory");
+ return null;
+ }
+
+ return new StringBuilder().append(workDir.getAbsolutePath())
+ .append(File.separator)
+ .append(SliderKeys.SECURITY_DIR)
+ .append(File.separator)
+ .append(SliderKeys.KEYSTORE_FILE_NAME).toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignCertResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignCertResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignCertResponse.java
new file mode 100644
index 0000000..8437d88
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignCertResponse.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.security;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlType;
+
+/**
+ *
+ * Sign certificate response data model.
+ *
+ */
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+@XmlType(name = "", propOrder = {})
+public class SignCertResponse {
+
+ public static final String ERROR_STATUS = "ERROR";
+ public static final String OK_STATUS = "OK";
+
+ @XmlElement
+ private String result;
+ @XmlElement
+ private String signedCa;
+ @XmlElement
+ private String message;
+
+ public String getResult() {
+ return result;
+ }
+ public void setResult(String result) {
+ this.result = result;
+ }
+ public String getSignedCa() {
+ return signedCa;
+ }
+ public void setSignedCa(String signedCa) {
+ this.signedCa = signedCa;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+ public void setMessage(String message) {
+ this.message = message;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignMessage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignMessage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignMessage.java
new file mode 100644
index 0000000..4bccb87
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignMessage.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.security;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlType;
+
+/**
+ *
+ * Sign certificate request data model.
+ *
+ */
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+@XmlType(name = "", propOrder = {})
+public class SignMessage {
+
+ @XmlElement
+ private String csr;
+ @XmlElement
+ private String passphrase;
+ public String getCsr() {
+ return csr;
+ }
+ public void setCsr(String csr) {
+ this.csr = csr;
+ }
+ public String getPassphrase() {
+ return passphrase;
+ }
+ public void setPassphrase(String passphrase) {
+ this.passphrase = passphrase;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/StoresGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/StoresGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/StoresGenerator.java
new file mode 100644
index 0000000..226250f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/StoresGenerator.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.services.security;
+
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.SliderException;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ *
+ */
+public class StoresGenerator {
+
+ static CertificateManager certMgr = new CertificateManager();
+ private static SecurityStoreGenerator[] GENERATORS = {
+ new KeystoreGenerator(certMgr), new TruststoreGenerator(certMgr)
+ };
+
+ public static SecurityStore[] generateSecurityStores(String hostname,
+ String containerId,
+ String role,
+ AggregateConf instanceDefinition,
+ MapOperations compOps)
+ throws SliderException, IOException {
+ //discover which stores need generation based on the passwords configured
+ List<SecurityStore> files = new ArrayList<SecurityStore>();
+ for (SecurityStoreGenerator generator : GENERATORS) {
+ if (generator.isStoreRequested(compOps)) {
+ SecurityStore store = generator.generate(hostname,
+ containerId,
+ instanceDefinition,
+ compOps,
+ role);
+ if (store != null) {
+ files.add(store);
+ }
+ }
+ }
+
+ if (files.isEmpty()) {
+ throw new SliderException("Security stores were requested but none were "
+ + "generated. Check the AM logs and ensure "
+ + "passwords are configured for the components "
+ + "requiring the stores.");
+ }
+ return files.toArray(new SecurityStore[files.size()]);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/TruststoreGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/TruststoreGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/TruststoreGenerator.java
new file mode 100644
index 0000000..d16dcbd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/TruststoreGenerator.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.services.security;
+
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.SliderException;
+
+import java.io.IOException;
+
+/**
+ *
+ */
+public class TruststoreGenerator extends AbstractSecurityStoreGenerator {
+
+
+ public TruststoreGenerator(CertificateManager certificateMgr) {
+ super(certificateMgr);
+ }
+
+ @Override
+ public SecurityStore generate(String hostname, String containerId,
+ AggregateConf instanceDefinition,
+ MapOperations compOps, String role)
+ throws SliderException, IOException {
+ SecurityStore truststore = null;
+ String password = getStorePassword(
+ instanceDefinition.getAppConf().credentials, compOps, role);
+ if (password != null) {
+ truststore = certificateMgr.generateContainerTruststore(containerId,
+ role, password);
+ }
+ return truststore;
+ }
+
+ @Override
+ String getPassword(MapOperations compOps) {
+ return compOps.get(
+ compOps.get(SliderKeys.COMP_TRUSTSTORE_PASSWORD_PROPERTY_KEY));
+ }
+
+ @Override
+ String getAlias(MapOperations compOps) {
+ return compOps.getOption(SliderKeys.COMP_TRUSTSTORE_PASSWORD_ALIAS_KEY,
+ SliderKeys.COMP_TRUSTSTORE_PASSWORD_ALIAS_DEFAULT);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
new file mode 100644
index 0000000..1622309
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.utility;
+
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.slider.common.tools.ConfigHelper;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.zk.ZookeeperUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base service for the standard slider client/server services
+ */
+public abstract class AbstractSliderLaunchedService extends
+ LaunchedWorkflowCompositeService {
+ private static final Logger log =
+ LoggerFactory.getLogger(AbstractSliderLaunchedService.class);
+
+ protected AbstractSliderLaunchedService(String name) {
+ super(name);
+ // make sure all the yarn configs get loaded
+ YarnConfiguration conf = new YarnConfiguration();
+ ConfigHelper.registerDeprecatedConfigItems();
+ }
+
+ /**
+ * look up the registry quorum from the config
+ * @return the quorum string
+ * @throws BadConfigException if it is not there or invalid
+ */
+ public String lookupZKQuorum() throws BadConfigException {
+
+ String registryQuorum = getConfig().get(RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
+
+ // though if neither is set: trouble
+ if (SliderUtils.isUnset(registryQuorum)) {
+ throw new BadConfigException(
+ "No Zookeeper quorum provided in the"
+ + " configuration property " + RegistryConstants.KEY_REGISTRY_ZK_QUORUM
+ );
+ }
+ ZookeeperUtils.splitToHostsAndPortsStrictly(registryQuorum);
+ return registryQuorum;
+ }
+
+ /**
+ * Create, adopt ,and start the YARN registration service
+ * @return the registry operations service, already deployed as a child
+ * of the AbstractSliderLaunchedService instance.
+ */
+ public RegistryOperations startRegistryOperationsService()
+ throws BadConfigException {
+
+ // push back the slider registry entry if needed
+ String quorum = lookupZKQuorum();
+ RegistryOperations registryWriterService =
+ createRegistryOperationsInstance();
+ deployChildService(registryWriterService);
+ return registryWriterService;
+ }
+
+ /**
+ * Create the registry operations instance. This is to allow
+ * subclasses to instantiate a subclass service
+ * @return an instance to match to the lifecycle of this service
+ */
+ protected RegistryOperations createRegistryOperationsInstance() {
+ return RegistryOperationsFactory.createInstance("YarnRegistry", getConfig());
+ }
+
+ /**
+ * Utility method to require an argument to be set (non null, non-empty)
+ * @param argname argument name
+ * @param value value
+ * @throws BadCommandArgumentsException if the condition is not met
+ */
+ protected static void requireArgumentSet(String argname, String value)
+ throws BadCommandArgumentsException {
+ require(isSet(value), "Required argument %s missing", argname );
+ }
+
+ /**
+ * Require a condition to hold; throw {@link BadCommandArgumentsException} if not.
+ * The exception text is the formatted message.
+ * @param condition condition
+ * @param message string to format
+ * @param args list of arguments to format.
+ * @throws BadCommandArgumentsException
+ */
+ protected static void require(boolean condition, String message,
+ Object... args)
+ throws BadCommandArgumentsException {
+ if (!condition) {
+ throw new BadCommandArgumentsException(message, args);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java
new file mode 100644
index 0000000..40ceab8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.utility;
+
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.service.ServiceStateChangeListener;
+
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Wait for a service to stop.
+ *
+ * WARNING: the notification may come in as soon as the service enters
+ * the stopped state: it may take some time for the actual stop operation
+ * to complete.
+ */
+public class EndOfServiceWaiter implements ServiceStateChangeListener {
+
+ private final AtomicBoolean finished = new AtomicBoolean(false);
+ private final String name;
+ private Service service;
+
+ /**
+ * Wait for a service; use the service name as this instance's name
+ * @param service service
+ */
+ public EndOfServiceWaiter(Service service) {
+ this(service.getName(), service);
+ }
+
+
+ /**
+ * Wait for a service
+ * @param name name for messages
+ * @param service service
+ */
+ public EndOfServiceWaiter(String name, Service service) {
+ this.name = name;
+ this.service = service;
+ service.registerServiceListener(this);
+ }
+
+ public synchronized void waitForServiceToStop(long timeout) throws
+ InterruptedException, TimeoutException {
+ service.waitForServiceToStop(timeout);
+ if (!finished.get()) {
+ wait(timeout);
+ if (!finished.get()) {
+ throw new TimeoutException(name
+ + " did not finish after " + timeout +
+ " milliseconds");
+ }
+ }
+ }
+
+ /**
+ * Wait for service state change callbacks; notify self if the service has
+ * now stopped
+ * @param service service
+ */
+ @Override
+ public synchronized void stateChanged(Service service) {
+ if (service.isInState(Service.STATE.STOPPED)) {
+ finished.set(true);
+ notify();
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java
new file mode 100644
index 0000000..bcd1969
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/LaunchedWorkflowCompositeService.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.utility;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.Service;
+import org.apache.slider.core.main.LauncherExitCodes;
+import org.apache.slider.core.main.RunService;
+import org.apache.slider.server.services.workflow.WorkflowCompositeService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This is a workflow compositoe service which can be launched from the CLI
+ * ... catches the arguments and implements a stub runService operation.
+ */
+public class LaunchedWorkflowCompositeService extends WorkflowCompositeService
+ implements RunService {
+ private static final Logger log = LoggerFactory.getLogger(
+ LaunchedWorkflowCompositeService.class);
+ private String[] argv;
+
+ public LaunchedWorkflowCompositeService(String name) {
+ super(name);
+ }
+
+ public LaunchedWorkflowCompositeService(String name, Service... children) {
+ super(name, children);
+ }
+
+ /**
+ * Implementation of set-ness, groovy definition of true/false for a string
+ * @param s
+ * @return true iff the string is non-null and non-empty
+ */
+ protected static boolean isUnset(String s) {
+ return StringUtils.isEmpty(s);
+ }
+
+ protected static boolean isSet(String s) {
+ return StringUtils.isNotEmpty(s);
+ }
+
+ protected String[] getArgv() {
+ return argv;
+ }
+
+ /**
+ * Pre-init argument binding
+ * @param config the initial configuration build up by the
+ * service launcher.
+ * @param args argument list list of arguments passed to the command line
+ * after any launcher-specific commands have been stripped.
+ * @return the configuration
+ * @throws Exception
+ */
+ @Override
+ public Configuration bindArgs(Configuration config, String... args) throws
+ Exception {
+ this.argv = args;
+ if (log.isDebugEnabled()) {
+ log.debug("Binding {} Arguments:", args.length);
+
+ StringBuilder builder = new StringBuilder();
+ for (String arg : args) {
+ builder.append('"').append(arg).append("\" ");
+ }
+ log.debug(builder.toString());
+ }
+ return config;
+ }
+
+ @Override
+ public int runService() throws Throwable {
+ return LauncherExitCodes.EXIT_SUCCESS;
+ }
+
+ @Override
+ public synchronized void addService(Service service) {
+ Preconditions.checkArgument(service != null, "null service argument");
+ super.addService(service);
+ }
+
+ /**
+ * Run a child service -initing and starting it if this
+ * service has already passed those parts of its own lifecycle
+ * @param service the service to start
+ */
+ protected boolean deployChildService(Service service) {
+ service.init(getConfig());
+ addService(service);
+ if (isInState(STATE.STARTED)) {
+ service.start();
+ return true;
+ }
+ return false;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/PatternValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/PatternValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/PatternValidator.java
new file mode 100644
index 0000000..6ab9de6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/PatternValidator.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.utility;
+
+import org.apache.slider.server.appmaster.web.rest.RestPaths;
+
+import java.util.regex.Pattern;
+
+/**
+ * Utility class to validate strings against a predefined pattern.
+ */
+public class PatternValidator {
+
+ public static final String E_INVALID_NAME =
+ "Invalid name %s does not match the pattern pattern %s ";
+ private final Pattern valid;
+ private final String pattern;
+
+ public PatternValidator(String pattern) {
+ this.pattern = pattern;
+ valid = Pattern.compile(pattern);
+ }
+
+ /**
+ * Validate the name -restricting it to the set defined in
+ * {@link RestPaths#PUBLISHED_CONFIGURATION_REGEXP}
+ * @param name name to validate
+ * @throws IllegalArgumentException if not a valid name
+ */
+ public void validate(String name) {
+ if (!matches(name)) {
+ throw new IllegalArgumentException(
+ String.format(E_INVALID_NAME, name, pattern));
+ }
+ }
+
+ /**
+ * Query to see if the pattern matches
+ * @param name name to validate
+ * @return true if the string matches the pattern
+ */
+ public boolean matches(String name) {
+ return valid.matcher(name).matches();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/WebAppService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/WebAppService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/WebAppService.java
new file mode 100644
index 0000000..ebfcb99
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/WebAppService.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.utility;
+
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.webapp.WebApp;
+
+/**
+ * Contains a webapp reference and stops it in teardown if non-null
+ * <p>
+ * It does not start the application.
+ * Access to the field is not synchronized across threads; it is the
+ * responsibility of the caller.
+ */
+public class WebAppService<T extends WebApp> extends AbstractService {
+
+ private volatile T webApp;
+
+ public WebAppService(String name) {
+ super(name);
+ }
+
+ public WebAppService(String name, T app) {
+ super(name);
+ webApp = app;
+ }
+
+ public T getWebApp() {
+ return webApp;
+ }
+
+ public void setWebApp(T webApp) {
+ this.webApp = webApp;
+ }
+
+
+ @Override
+ protected void serviceStart() throws Exception {
+
+ }
+
+ /**
+ * Stop operation stops the webapp; sets the reference to null
+ * @throws Exception
+ */
+ @Override
+ protected void serviceStop() throws Exception {
+ if (webApp != null) {
+ webApp.stop();
+ webApp = null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java
new file mode 100644
index 0000000..8b711aa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ClosingService.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.workflow;
+
+import org.apache.hadoop.service.AbstractService;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+/**
+ * Service that closes the closeable supplied during shutdown, if not null.
+ *
+ * As the Service interface itself extends Closeable, this service
+ * can be used to shut down other services if desired.
+ */
+public class ClosingService<C extends Closeable> extends AbstractService {
+
+ private C closeable;
+
+ public ClosingService(String name) {
+ super(name);
+ }
+
+ /**
+ * Construct an instance of the service
+ * @param name service name
+ * @param closeable closeable to close (may be null)
+ */
+ public ClosingService(String name,
+ C closeable) {
+ super(name);
+ this.closeable = closeable;
+ }
+
+ /**
+ * Construct an instance of the service, using the default name
+ * @param closeable closeable to close (may be null)
+ */
+ public ClosingService(C closeable) {
+ this("ClosingService", closeable);
+ }
+
+
+ /**
+ * Get the closeable
+ * @return the closeable
+ */
+ public synchronized C getCloseable() {
+ return closeable;
+ }
+
+ /**
+ * Set or update the closeable.
+ * @param closeable
+ */
+ public synchronized void setCloseable(C closeable) {
+ this.closeable = closeable;
+ }
+
+ /**
+ * Stop routine will close the closeable -if not null - and set the
+ * reference to null afterwards
+ * This operation does raise any exception on the close, though it does
+ * record it
+ */
+ @Override
+ protected void serviceStop() {
+ C target = getCloseable();
+ if (target != null) {
+ try {
+ target.close();
+ } catch (IOException ioe) {
+ noteFailure(ioe);
+ }
+ setCloseable(null);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8cab88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
new file mode 100644
index 0000000..352be49
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
@@ -0,0 +1,301 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.workflow;
+
+import org.apache.hadoop.service.ServiceStateException;
+import org.apache.slider.core.main.ServiceLaunchException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Service wrapper for an external program that is launched and can/will terminate.
+ * This service is notified when the subprocess terminates, and stops itself
+ * and converts a non-zero exit code into a failure exception.
+ *
+ * <p>
+ * Key Features:
+ * <ol>
+ * <li>The property {@link #executionTimeout} can be set to set a limit
+ * on the duration of a process</li>
+ * <li>Output is streamed to the output logger provided</li>.
+ * <li>The most recent lines of output are saved to a linked list</li>.
+ * <li>A synchronous callback, {@link LongLivedProcessLifecycleEvent}, is raised on the start
+ * and finish of a process.</li>
+ * </ol>
+ *
+ * Usage:
+ * <p></p>
+ * The service can be built in the constructor, {@link #ForkedProcessService(String, Map, List)},
+ * or have its simple constructor used to instantiate the service, then the
+ * {@link #build(Map, List)} command used to define the environment variables
+ * and list of commands to execute. One of these two options MUST be exercised
+ * before calling the services's {@link #start()} method.
+ * <p></p>
+ * The forked process is executed in the service's {@link #serviceStart()} method;
+ * if still running when the service is stopped, {@link #serviceStop()} will
+ * attempt to stop it.
+ * <p></p>
+ *
+ * The service delegates process execution to {@link LongLivedProcess},
+ * receiving callbacks via the {@link LongLivedProcessLifecycleEvent}.
+ * When the service receives a callback notifying that the process has completed,
+ * it calls its {@link #stop()} method. If the error code was non-zero,
+ * the service is logged as having failed.
+ */
+public class ForkedProcessService
+ extends WorkflowExecutorService<ExecutorService>
+ implements LongLivedProcessLifecycleEvent, Runnable {
+
+ /**
+ * Log for the forked master process
+ */
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ForkedProcessService.class);
+
+ private final AtomicBoolean processTerminated = new AtomicBoolean(false);
+ private boolean processStarted = false;
+ private LongLivedProcess process;
+ private int executionTimeout = -1;
+ private int timeoutCode = 1;
+ /**
+ log to log to; defaults to this service log
+ */
+ private Logger processLog = LOG;
+
+ /**
+ * Exit code set when the spawned process exits
+ */
+ private AtomicInteger exitCode = new AtomicInteger(0);
+
+ /**
+ * Create an instance of the service
+ * @param name a name
+ */
+ public ForkedProcessService(String name) {
+ super(name);
+ }
+
+ /**
+ * Create an instance of the service, set up the process
+ * @param name a name
+ * @param commandList list of commands is inserted on the front
+ * @param env environment variables above those generated by
+ * @throws IOException IO problems
+ */
+ public ForkedProcessService(String name,
+ Map<String, String> env,
+ List<String> commandList) throws IOException {
+ super(name);
+ build(env, commandList);
+ }
+
+ @Override //AbstractService
+ protected void serviceStart() throws Exception {
+ if (process == null) {
+ throw new ServiceStateException("Process not yet configured");
+ }
+ //now spawn the process -expect updates via callbacks
+ process.start();
+ }
+
+ @Override //AbstractService
+ protected void serviceStop() throws Exception {
+ completed();
+ stopForkedProcess();
+ }
+
+ private void stopForkedProcess() {
+ if (process != null) {
+ process.stop();
+ }
+ }
+
+ /**
+ * Set the process log. This may be null for "do not log"
+ * @param processLog process log
+ */
+ public void setProcessLog(Logger processLog) {
+ this.processLog = processLog;
+ process.setProcessLog(processLog);
+ }
+
+ /**
+ * Set the timeout by which time a process must have finished -or -1 for forever
+ * @param timeout timeout in milliseconds
+ */
+ public void setTimeout(int timeout, int code) {
+ this.executionTimeout = timeout;
+ this.timeoutCode = code;
+ }
+
+ /**
+ * Build the process to execute when the service is started
+ * @param commandList list of commands is inserted on the front
+ * @param env environment variables above those generated by
+ * @throws IOException IO problems
+ */
+ public void build(Map<String, String> env,
+ List<String> commandList)
+ throws IOException {
+ assert process == null;
+
+ process = new LongLivedProcess(getName(), processLog, commandList);
+ process.setLifecycleCallback(this);
+ //set the env variable mapping
+ process.putEnvMap(env);
+ }
+
+ @Override // notification from executed process
+ public synchronized void onProcessStarted(LongLivedProcess process) {
+ LOG.debug("Process has started");
+ processStarted = true;
+ if (executionTimeout > 0) {
+ setExecutor(ServiceThreadFactory.singleThreadExecutor(getName(), true));
+ execute(this);
+ }
+ }
+
+ @Override // notification from executed process
+ public void onProcessExited(LongLivedProcess process,
+ int uncorrected,
+ int code) {
+ try {
+ synchronized (this) {
+ completed();
+ //note whether or not the service had already stopped
+ LOG.debug("Process has exited with exit code {}", code);
+ if (code != 0) {
+ reportFailure(code, getName() + " failed with code " + code);
+ }
+ }
+ } finally {
+ stop();
+ }
+ }
+
+ private void reportFailure(int code, String text) {
+ //error
+ ServiceLaunchException execEx = new ServiceLaunchException(code, text);
+ LOG.debug("Noting failure", execEx);
+ noteFailure(execEx);
+ }
+
+ /**
+ * handle timeout response by escalating it to a failure
+ */
+ @Override
+ public void run() {
+ try {
+ synchronized (processTerminated) {
+ if (!processTerminated.get()) {
+ processTerminated.wait(executionTimeout);
+ }
+ }
+
+ } catch (InterruptedException e) {
+ //assume signalled; exit
+ }
+ //check the status; if the marker isn't true, bail
+ if (!processTerminated.getAndSet(true)) {
+ LOG.info("process timeout: reporting error code {}", timeoutCode);
+
+ //timeout
+ if (isInState(STATE.STARTED)) {
+ //trigger a failure
+ stopForkedProcess();
+ }
+ reportFailure(timeoutCode, getName() + ": timeout after " + executionTimeout
+ + " millis: exit code =" + timeoutCode);
+ }
+ }
+
+ /**
+ * Note the process as having completed.
+ * The process marked as terminated
+ * -and anything synchronized on <code>processTerminated</code>
+ * is notified
+ */
+ protected void completed() {
+ processTerminated.set(true);
+ synchronized (processTerminated) {
+ processTerminated.notify();
+ }
+ }
+
+ public boolean isProcessTerminated() {
+ return processTerminated.get();
+ }
+
+ public synchronized boolean isProcessStarted() {
+ return processStarted;
+ }
+
+ /**
+ * Is a process running: between started and terminated
+ * @return true if the process is up.
+ */
+ public synchronized boolean isProcessRunning() {
+ return processStarted && !isProcessTerminated();
+ }
+
+
+ public Integer getExitCode() {
+ return process.getExitCode();
+ }
+
+ public int getExitCodeSignCorrected() {
+ Integer exitCode = process.getExitCodeSignCorrected();
+ if (exitCode == null) return -1;
+ return exitCode;
+ }
+
+ /**
+ * Get the recent output from the process, or [] if not defined
+ * @return a possibly empty list
+ */
+ public List<String> getRecentOutput() {
+ return process != null
+ ? process.getRecentOutput()
+ : new LinkedList<String>();
+ }
+
+ /**
+ * Get the recent output from the process, or [] if not defined
+ *
+ * @param finalOutput flag to indicate "wait for the final output of the process"
+ * @param duration the duration, in ms,
+ * to wait for recent output to become non-empty
+ * @return a possibly empty list
+ */
+ public List<String> getRecentOutput(boolean finalOutput, int duration) {
+ if (process == null) {
+ return new LinkedList<>();
+ }
+ return process.getRecentOutput(finalOutput, duration);
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org