You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/10/22 13:34:40 UTC
[2/9] git commit: SLIDER-517 enable certificate and key localization
for containers
SLIDER-517 enable certificate and key localization for containers
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/46e131df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/46e131df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/46e131df
Branch: refs/heads/feature/SLIDER-460-stderr
Commit: 46e131df617c2461ef88905d083a2802468f0494
Parents: 0fa1141
Author: Jon Maron <jm...@hortonworks.com>
Authored: Tue Oct 21 14:24:47 2014 -0400
Committer: Jon Maron <jm...@hortonworks.com>
Committed: Tue Oct 21 14:24:47 2014 -0400
----------------------------------------------------------------------
.../slider/providers/agent/AgentKeys.java | 2 +
.../providers/agent/AgentProviderService.java | 155 +++++++++++++++----
.../server/appmaster/SliderAppMaster.java | 30 ++++
.../appmaster/web/rest/agent/AgentWebApp.java | 4 +-
.../services/security/CertificateManager.java | 74 ++++++++-
5 files changed, 229 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46e131df/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index 2612865..9d5eb3d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -97,6 +97,8 @@ public interface AgentKeys {
String HEARTBEAT_MONITOR_INTERVAL = "heartbeat.monitor.interval";
String AGENT_INSTANCE_DEBUG_DATA = "agent.instance.debug.data";
String AGENT_OUT_FILE = "slider-agent.out";
+ String KEY_AGENT_TWO_WAY_SSL_ENABLED = "ssl.server.client.auth";
+ String CERT_FILE_LOCALIZATION_PATH = "certs/ca.crt";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46e131df/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 19ec015..d0ae5be 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -20,18 +20,22 @@ package org.apache.slider.providers.agent;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.ProtocolTypes;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.registry.client.types.Endpoint;
-import org.apache.hadoop.registry.client.types.ProtocolTypes;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterDescriptionKeys;
import org.apache.slider.api.ClusterNode;
@@ -39,6 +43,7 @@ import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.OptionKeys;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.StatusKeys;
+import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.tools.SliderFileSystem;
@@ -88,6 +93,7 @@ import org.apache.slider.server.appmaster.web.rest.agent.Register;
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse;
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus;
import org.apache.slider.server.appmaster.web.rest.agent.StatusCommand;
+import org.apache.slider.server.services.security.CertificateManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -103,7 +109,6 @@ import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
@@ -378,33 +383,15 @@ public class AgentProviderService extends AbstractProviderService implements
}
if (SliderUtils.isHadoopClusterSecure(getConfig())) {
- String keytabPathOnHost = instanceDefinition.getAppConfOperations()
- .getComponent(SliderKeys.COMPONENT_AM).get(
- SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
- if (SliderUtils.isUnset(keytabPathOnHost)) {
- String amKeytabName = instanceDefinition.getAppConfOperations()
- .getComponent(SliderKeys.COMPONENT_AM).get(
- SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
- String keytabDir = instanceDefinition.getAppConfOperations()
- .getComponent(SliderKeys.COMPONENT_AM).get(
- SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR);
- // we need to localize the keytab files in the directory
- Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null,
- clusterName);
- FileStatus[] keytabs = fileSystem.getFileSystem().listStatus(keytabDirPath);
- LocalResource keytabRes;
- for (FileStatus keytab : keytabs) {
- if (!amKeytabName.equals(keytab.getPath().getName())
- && keytab.getPath().getName().endsWith(".keytab")) {
- log.info("Localizing keytab {}", keytab.getPath().getName());
- keytabRes = fileSystem.createAmResource(keytab.getPath(),
- LocalResourceType.FILE);
- launcher.addLocalResource(SliderKeys.KEYTAB_DIR + "/" +
- keytab.getPath().getName(),
- keytabRes);
- }
- }
- }
+ localizeServiceKeytabs(launcher, instanceDefinition, fileSystem);
+ }
+
+ MapOperations amComponent = instanceDefinition.
+ getAppConfOperations().getComponent(SliderKeys.COMPONENT_AM);
+ boolean twoWayEnabled = amComponent != null ? Boolean.valueOf(amComponent.
+ getOptionBool(AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED, false)) : false;
+ if (twoWayEnabled) {
+ localizeContainerSSLResources(launcher, container, fileSystem);
}
//add the configuration resources
@@ -442,6 +429,112 @@ public class AgentProviderService extends AbstractProviderService implements
getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME)));
}
+ private void localizeContainerSSLResources(ContainerLauncher launcher,
+ Container container,
+ SliderFileSystem fileSystem)
+ throws SliderException {
+ try {
+ // localize server cert
+ Path certsDir = new Path(fileSystem.buildClusterDirPath(
+ getClusterName()), "certs");
+ LocalResource certResource = fileSystem.createAmResource(
+ new Path(certsDir, SliderKeys.CRT_FILE_NAME),
+ LocalResourceType.FILE);
+ launcher.addLocalResource(AgentKeys.CERT_FILE_LOCALIZATION_PATH,
+ certResource);
+
+ // generate and localize agent cert
+ CertificateManager certMgr = new CertificateManager();
+ String hostname = container.getNodeId().getHost();
+ String containerId = container.getId().toString();
+ certMgr.generateAgentCertificate(hostname, containerId);
+ LocalResource agentCertResource = fileSystem.createAmResource(
+ uploadSecurityResource(
+ CertificateManager.getAgentCertficateFilePath(containerId),
+ fileSystem), LocalResourceType.FILE);
+ // still using hostname as file name on the agent side, but the files
+ // do end up under the specific container's file space
+ launcher.addLocalResource("certs/" + hostname + ".crt",
+ agentCertResource);
+ LocalResource agentKeyResource = fileSystem.createAmResource(
+ uploadSecurityResource(
+ CertificateManager.getAgentKeyFilePath(containerId), fileSystem),
+ LocalResourceType.FILE);
+ launcher.addLocalResource("certs/" + hostname + ".key",
+ agentKeyResource);
+
+ } catch (Exception e) {
+ throw new SliderException(SliderExitCodes.EXIT_DEPLOYMENT_FAILED, e,
+ "Unable to localize certificates. Two-way SSL cannot be enabled");
+ }
+ }
+
+ private Path uploadSecurityResource(File resource, SliderFileSystem fileSystem)
+ throws IOException {
+ Path certsDir = new Path(fileSystem.buildClusterDirPath(getClusterName()),
+ "certs");
+ if (!fileSystem.getFileSystem().exists(certsDir)) {
+ fileSystem.getFileSystem().mkdirs(certsDir,
+ new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
+ }
+ Path destPath = new Path(certsDir, resource.getName());
+ if (!fileSystem.getFileSystem().exists(destPath)) {
+ FSDataOutputStream os = fileSystem.getFileSystem().create(destPath);
+ byte[] contents = FileUtils.readFileToByteArray(resource);
+ os.write(contents, 0, contents.length);
+
+ os.flush();
+ os.close();
+ log.info("Uploaded {} to localization path {}", resource, destPath);
+ }
+
+ while (!fileSystem.getFileSystem().exists(destPath)) {
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ // ignore
+ }
+ }
+
+ fileSystem.getFileSystem().setPermission(destPath,
+ new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE));
+
+ return destPath;
+ }
+
+ private void localizeServiceKeytabs(ContainerLauncher launcher,
+ AggregateConf instanceDefinition,
+ SliderFileSystem fileSystem)
+ throws IOException {
+ String keytabPathOnHost = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(
+ SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
+ if (SliderUtils.isUnset(keytabPathOnHost)) {
+ String amKeytabName = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(
+ SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
+ String keytabDir = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(
+ SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR);
+ // we need to localize the keytab files in the directory
+ Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null,
+ getClusterName());
+ FileStatus[] keytabs = fileSystem.getFileSystem().listStatus(keytabDirPath);
+ LocalResource keytabRes;
+ for (FileStatus keytab : keytabs) {
+ if (!amKeytabName.equals(keytab.getPath().getName())
+ && keytab.getPath().getName().endsWith(".keytab")) {
+ log.info("Localizing keytab {}", keytab.getPath().getName());
+ keytabRes = fileSystem.createAmResource(keytab.getPath(),
+ LocalResourceType.FILE);
+ launcher.addLocalResource(SliderKeys.KEYTAB_DIR + "/" +
+ keytab.getPath().getName(),
+ keytabRes);
+ }
+ }
+ }
+ }
+
/**
* build the zookeeper registry path.
*
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46e131df/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 0a7fe33..d696f45 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -25,6 +25,8 @@ import com.google.protobuf.BlockingService;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
@@ -108,6 +110,7 @@ import org.apache.slider.providers.ProviderCompleted;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.providers.SliderProviderFactory;
+import org.apache.slider.providers.agent.AgentKeys;
import org.apache.slider.providers.slideram.SliderAMClientProvider;
import org.apache.slider.providers.slideram.SliderAMProviderService;
import org.apache.slider.server.appmaster.actions.ActionKillContainer;
@@ -679,6 +682,12 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
instanceDefinition.getAppConfOperations()
.getComponent(SliderKeys.COMPONENT_AM));
+ if (Boolean.valueOf(instanceDefinition.
+ getAppConfOperations().getComponent(SliderKeys.COMPONENT_AM).
+ getOptionBool(AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED, false))) {
+ uploadServerCertForLocalization(clustername, fs);
+ }
+
startAgentWebApp(appInformation, serviceConf);
webApp = new SliderAMWebApp(registryOperations);
@@ -891,6 +900,27 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
return finish();
}
+ private void uploadServerCertForLocalization(String clustername,
+ SliderFileSystem fs)
+ throws IOException {
+ Path certsDir = new Path(fs.buildClusterDirPath(clustername),
+ "certs");
+ if (!fs.getFileSystem().exists(certsDir)) {
+ fs.getFileSystem().mkdirs(certsDir,
+ new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
+ }
+ Path destPath = new Path(certsDir, SliderKeys.CRT_FILE_NAME);
+ if (!fs.getFileSystem().exists(destPath)) {
+ fs.getFileSystem().copyFromLocalFile(
+ new Path(CertificateManager.getServerCertficateFilePath().getAbsolutePath()),
+ destPath);
+ log.info("Uploaded server cert to localization path {}", destPath);
+ }
+
+ fs.getFileSystem().setPermission(destPath,
+ new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE));
+ }
+
protected void login(String principal, File localKeytabFile)
throws IOException, SliderException {
UserGroupInformation.loginUserFromKeytab(principal,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46e131df/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
index 8aac490..f8d7b88 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
@@ -23,6 +23,7 @@ import com.sun.jersey.spi.container.servlet.ServletContainer;
import com.sun.jersey.spi.container.servlet.WebConfig;
import com.sun.jersey.spi.inject.SingletonTypeInjectableProvider;
import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.providers.agent.AgentKeys;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
import org.apache.slider.server.services.security.SecurityUtils;
@@ -82,7 +83,8 @@ public class AgentWebApp implements Closeable {
SslSelectChannelConnector ssl1WayConnector = createSSLConnector(false);
SslSelectChannelConnector ssl2WayConnector =
createSSLConnector(Boolean.valueOf(
- configsMap.getOption("ssl.server.client.auth","false")));
+ configsMap.getOption(AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED,
+ "false")));
agentServer.setConnectors(new Connector[]{ssl1WayConnector,
ssl2WayConnector});
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/46e131df/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java b/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
index d200033..8934892 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
@@ -55,7 +55,11 @@ public class CertificateManager {
"-passin pass:{3} -cert {0}/{5}";
private static final String SIGN_AGENT_CRT = "openssl ca -config " +
"{0}/ca.config -in {0}/{1} -out {0}/{2} -batch -passin pass:{3} " +
- "-keyfile {0}/{4} -cert {0}/{5}"; /**
+ "-keyfile {0}/{4} -cert {0}/{5}";
+ private static final String GEN_AGENT_KEY="openssl req -new -newkey " +
+ "rsa:1024 -nodes -keyout {0}/{2}.key -subj /OU={1}/CN={2} -out {0}/{2}.csr";
+
+ /**
* Verify that root certificate exists, generate it otherwise.
*/
public void initRootCert(MapOperations compOperations) {
@@ -148,7 +152,7 @@ public class CertificateManager {
SecurityUtils.logOpenSslExitCode(command, process.exitValue());
exitCode = process.exitValue();
if (exitCode != 0) {
- throw new SliderException(exitCode, "Error running command {}", command);
+ throw new SliderException(exitCode, "Error running command %s", command);
}
} catch (InterruptedException e) {
e.printStackTrace();
@@ -169,6 +173,23 @@ public class CertificateManager {
}
+ public synchronized void generateAgentCertificate(String agentHostname, String containerId) {
+ LOG.info("Generation of agent certificate for {}", agentHostname);
+
+ String srvrKstrDir = SecurityUtils.getSecurityDir();
+ Object[] scriptArgs = {srvrKstrDir, agentHostname, containerId};
+
+ try {
+ String command = MessageFormat.format(GEN_AGENT_KEY, scriptArgs);
+ runCommand(command);
+
+ signAgentCertificate(containerId);
+
+ } catch (SliderException e) {
+ LOG.error("Error generating the agent certificate", e);
+ }
+ }
+
private void generateServerCertificate(){
LOG.info("Generation of server certificate");
@@ -205,8 +226,7 @@ public class CertificateManager {
* @return string with server certificate content
*/
public String getServerCert() {
- File certFile = new File(SecurityUtils.getSecurityDir() +
- File.separator + SliderKeys.CRT_FILE_NAME);
+ File certFile = getServerCertficateFilePath();
String srvrCrtContent = null;
try {
srvrCrtContent = FileUtils.readFileToString(certFile);
@@ -216,6 +236,21 @@ public class CertificateManager {
return srvrCrtContent;
}
+ public static File getServerCertficateFilePath() {
+ return new File(SecurityUtils.getSecurityDir() +
+ File.separator + SliderKeys.CRT_FILE_NAME);
+ }
+
+ public static File getAgentCertficateFilePath(String containerId) {
+ return new File(SecurityUtils.getSecurityDir() +
+ File.separator + containerId + ".crt");
+ }
+
+ public static File getAgentKeyFilePath(String containerId) {
+ return new File(SecurityUtils.getSecurityDir() +
+ File.separator + containerId + ".key");
+ }
+
/**
* Signs agent certificate
* Adds agent certificate to server keystore
@@ -302,4 +337,35 @@ public class CertificateManager {
//LOG.info(ShellCommandUtil.getOpenSslCommandResult(command, commandExitCode));
return response;
}
+
+ private String signAgentCertificate (String containerId)
+ throws SliderException {
+ String srvrKstrDir = SecurityUtils.getSecurityDir();
+ String srvrCrtPass = SecurityUtils.getKeystorePass();
+ String srvrCrtName = SliderKeys.CRT_FILE_NAME;
+ String srvrKeyName = SliderKeys.KEY_FILE_NAME;
+ String agentCrtReqName = containerId + ".csr";
+ String agentCrtName = containerId + ".crt";
+
+ Object[] scriptArgs = {srvrKstrDir, agentCrtReqName, agentCrtName,
+ srvrCrtPass, srvrKeyName, srvrCrtName};
+
+ //Revoke previous agent certificate if exists
+ File agentCrtFile = new File(srvrKstrDir + File.separator + agentCrtName);
+
+ String command;
+ if (agentCrtFile.exists()) {
+ LOG.info("Revoking of " + containerId + " certificate.");
+ command = MessageFormat.format(REVOKE_AGENT_CRT, scriptArgs);
+ runCommand(command);
+ }
+
+ command = MessageFormat.format(SIGN_AGENT_CRT, scriptArgs);
+
+ LOG.debug(SecurityUtils.hideOpenSslPassword(command));
+ runCommand(command);
+
+ return agentCrtName;
+
+ }
}