You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/10/08 02:03:30 UTC
[1/8] git commit: SLIDER-488 excluded transitive commons-codec
dependency from commons-httpclient
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry bfbc67aff -> 032dacf4c
SLIDER-488 excluded transitive commons-codec dependency from commons-httpclient
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/80e8df0c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/80e8df0c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/80e8df0c
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 80e8df0c66e41a06943a5e7555062ce05a2e0d22
Parents: 70b4b75
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Mon Oct 6 09:21:32 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Mon Oct 6 09:21:32 2014 -0700
----------------------------------------------------------------------
pom.xml | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80e8df0c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6d35407..4df4971 100644
--- a/pom.xml
+++ b/pom.xml
@@ -600,6 +600,12 @@
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
<version>${httpclient.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- ======================================================== -->
[5/8] git commit: SLIDER-480. Add a fun test to ensure that
containers are restarted on nodes with correct labels
Posted by st...@apache.org.
SLIDER-480. Add a fun test to ensure that containers are restarted on nodes with correct labels
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0419a5a9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0419a5a9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0419a5a9
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 0419a5a90afea0e3e7eab0f2d5ea09ab5aad7f89
Parents: 961e170
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Mon Oct 6 21:12:36 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Mon Oct 6 21:12:36 2014 -0700
----------------------------------------------------------------------
.../AppsThroughAgentQueueAndLabelsIT.groovy | 26 ++++++++++++++++++++
1 file changed, 26 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0419a5a9/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
index f3f6612..a3b0ccb 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
@@ -29,6 +29,32 @@ import org.apache.slider.funtest.framework.SliderShell
import org.junit.After
import org.junit.Test
+/**
+ * SETUP FOR THE TEST
+ * Create valid labels, red and blue [yarn rmadmin -addLabels red,blue]
+ * Add nodes with label [yarn rmadmin -setNodeToLabels host1:blue]
+ * Perform refresh queue [yarn rmadmin -refreshQueues]
+ *
+ * Create a queue with access to labels - these are changes to capacity scheduler configuration
+ * Add a queue in addition to default
+ * yarn.scheduler.capacity.root.queues=default,labeled
+ * Provide capacity, take out from default
+ * yarn.scheduler.capacity.root.labeled.capacity=80
+ * yarn.scheduler.capacity.root.default.capacity=20
+ * Provide standard queue specs
+ * yarn.scheduler.capacity.root.labeled.state=RUNNING
+ * yarn.scheduler.capacity.root.labeled.maximum-capacity=80
+ * Have queue access the label
+ * yarn.scheduler.capacity.root.labeled.labels=red,blue
+ *
+ * After specifying the new configuration call refresh [yarn rmadmin -refreshQueues]
+ *
+ * See resources_queue_labels.json for label configuration required for the test
+ * Label expression for slider-appmaster is also the default for all containers
+ * if they do not specify own label expressions
+ * "yarn.label.expression":"red"
+ *
+ */
@CompileStatic
@Slf4j
public class AppsThroughAgentQueueAndLabelsIT extends AgentCommandTestBase
[8/8] git commit: SLIDER-365 "slider resolve" command
Posted by st...@apache.org.
SLIDER-365 "slider resolve" command
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/032dacf4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/032dacf4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/032dacf4
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 032dacf4cf1757302fffc59782572f574d185cdc
Parents: c09205f
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 7 17:03:21 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 7 17:03:21 2014 -0700
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 301 +++++++------------
.../apache/slider/client/SliderClientAPI.java | 249 +++++++++++++++
.../slider/common/params/ActionResolveArgs.java | 5 -
.../apache/slider/common/params/ClientArgs.java | 4 +-
.../apache/slider/common/tools/SliderUtils.java | 30 +-
.../TestStandaloneYarnRegistryAM.groovy | 40 ++-
6 files changed, 418 insertions(+), 211 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/032dacf4/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 185e77d..c5022ee 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
@@ -140,6 +141,7 @@ import java.io.StringWriter;
import java.io.Writer;
import java.net.InetSocketAddress;
import java.net.URISyntaxException;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -159,7 +161,7 @@ import static org.apache.slider.common.params.SliderActions.*;
*/
public class SliderClient extends AbstractSliderLaunchedService implements RunService,
- SliderExitCodes, SliderKeys, ErrorStrings {
+ SliderExitCodes, SliderKeys, ErrorStrings, SliderClientAPI {
private static final Logger log = LoggerFactory.getLogger(SliderClient.class);
private ClientArgs serviceArgs;
@@ -197,6 +199,16 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
new YarnConfiguration();
}
+ /**
+ * This is called <i>Before serviceInit is called</i>
+ * @param config the initial configuration build up by the
+ * service launcher.
+ * @param args argument list list of arguments passed to the command line
+ * after any launcher-specific commands have been stripped.
+ * @return the post-binding configuration to pass to the <code>init()</code>
+ * operation.
+ * @throws Exception
+ */
@Override
public Configuration bindArgs(Configuration config, String... args) throws Exception {
config = super.bindArgs(config, args);
@@ -218,9 +230,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
SliderUtils.forceLogin();
SliderUtils.initProcessSecurity(conf);
}
-
+ AbstractActionArgs coreAction = serviceArgs.getCoreAction();
+ if (coreAction.getHadoopServicesRequired()) {
+ initHadoopBinding();
+ }
super.serviceInit(conf);
-
}
/**
@@ -308,11 +322,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// choose the action
String action = serviceArgs.getAction();
-
- AbstractActionArgs coreAction = serviceArgs.getCoreAction();
- if (coreAction.getHadoopServicesRequired()) {
- initHadoopBinding();
- }
+
int exitCode = EXIT_SUCCESS;
String clusterName = serviceArgs.getClusterName();
// actions
@@ -358,9 +368,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
serviceArgs.getActionStatusArgs());
} else if (ACTION_UPDATE.equals(action)) {
exitCode = actionUpdate(clusterName, serviceArgs.getActionUpdateArgs());
-
} else if (ACTION_VERSION.equals(action)) {
-
exitCode = actionVersion();
} else {
throw new SliderException(EXIT_UNIMPLEMENTED,
@@ -490,11 +498,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return client;
}
- /**
- * Destroy a cluster. There's two race conditions here
- * #1 the cluster is started between verifying that there are no live
- * clusters of that name.
- */
+ @Override
public int actionDestroy(String clustername) throws YarnException,
IOException {
// verify that a live cluster isn't there
@@ -536,11 +540,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
- /**
- * AM to commit an asynchronous suicide
- */
+ @Override
public int actionAmSuicide(String clustername,
- ActionAMSuicideArgs args) throws
+ ActionAMSuicideArgs args) throws
YarnException,
IOException {
SliderClusterOperations clusterOperations =
@@ -549,13 +551,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
- /**
- * Get the provider for this cluster
- * @param provider the name of the provider
- * @return the provider instance
- * @throws SliderException problems building the provider
- */
- private AbstractClientProvider createClientProvider(String provider)
+ @Override
+ public AbstractClientProvider createClientProvider(String provider)
throws SliderException {
SliderProviderFactory factory =
SliderProviderFactory.createSliderProviderFactory(provider);
@@ -622,17 +619,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
}
- /**
- * Build up the cluster specification/directory
- *
- * @param clustername cluster name
- * @param buildInfo the arguments needed to build the cluster
- * @throws YarnException Yarn problems
- * @throws IOException other problems
- * @throws BadCommandArgumentsException bad arguments.
- */
+ @Override
public int actionBuild(String clustername,
- AbstractClusterBuildingActionArgs buildInfo) throws
+ AbstractClusterBuildingActionArgs buildInfo) throws
YarnException,
IOException {
@@ -640,24 +629,17 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
- /**
- * Upload application package to user home directory
- *
- * @param installPkgInfo the arguments needed to build the cluster
- * @throws YarnException Yarn problems
- * @throws IOException other problems
- * @throws BadCommandArgumentsException bad arguments.
- */
+ @Override
public int actionInstallPkg(ActionInstallPackageArgs installPkgInfo) throws
YarnException,
IOException {
Path srcFile = null;
- if (null == installPkgInfo.name || installPkgInfo.name.length() == 0) {
+ if (StringUtils.isEmpty(installPkgInfo.name )) {
throw new BadCommandArgumentsException("A valid application type name is required (e.g. HBASE).");
}
- if (null == installPkgInfo.packageURI || installPkgInfo.packageURI.length() == 0) {
+ if (StringUtils.isEmpty(installPkgInfo.packageURI)) {
throw new BadCommandArgumentsException("A valid application package location required.");
} else {
File pkgFile = new File(installPkgInfo.packageURI);
@@ -684,15 +666,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
- /**
- * Update the cluster specification
- *
- * @param clustername cluster name
- * @param buildInfo the arguments needed to update the cluster
- * @throws YarnException Yarn problems
- * @throws IOException other problems
- */
- public int actionUpdate(String clustername, AbstractClusterBuildingActionArgs buildInfo) throws
+ @Override
+ public int actionUpdate(String clustername,
+ AbstractClusterBuildingActionArgs buildInfo) throws
YarnException, IOException {
buildInstanceDefinition(clustername, buildInfo, true, true);
return EXIT_SUCCESS;
@@ -1011,10 +987,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
/**
*
- * @param clustername
- * @param clusterDirectory
- * @param instanceDefinition
- * @param debugAM
+ * @param clustername name of the cluster
+ * @param clusterDirectory cluster dir
+ * @param instanceDefinition the instance definition
+ * @param debugAM enable debug AM options
* @return the launched application
* @throws YarnException
* @throws IOException
@@ -1579,23 +1555,14 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return launchedApplication.monitorAppToState(desiredState, duration);
}
- /**
- * Get the report of a this application
- * @return the app report or null if it could not be found.
- * @throws IOException
- * @throws YarnException
- */
+ @Override
public ApplicationReport getApplicationReport() throws
IOException,
YarnException {
return getApplicationReport(applicationId);
}
- /**
- * Kill the submitted application via YARN
- * @throws YarnException
- * @throws IOException
- */
+ @Override
public boolean forceKillApplication(String reason)
throws YarnException, IOException {
if (applicationId != null) {
@@ -1616,10 +1583,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return YarnAppListClient.listInstances();
}
- /**
- * Implement the list action: list all nodes
- * @return exit code of 0 if a list was created
- */
+ @Override
@VisibleForTesting
public int actionList(String clustername) throws IOException, YarnException {
verifyBindingsDefined();
@@ -1657,10 +1621,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
log.info(SliderUtils.appReportToString(report, "\n"));
}
- /**
- * Implement the islive action: probe for a cluster of the given name existing
- * @return exit code
- */
+ @Override
@VisibleForTesting
public int actionFlex(String name, ActionFlexArgs args) throws YarnException, IOException {
verifyBindingsDefined();
@@ -1682,11 +1643,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return flex(name, roleInstances);
}
- /**
- * Test for a cluster existing probe for a cluster of the given name existing
- * in the filesystem. If the live param is set, it must be a live cluster
- * @return exit code
- */
+ @Override
@VisibleForTesting
public int actionExists(String name, boolean checkLive) throws YarnException, IOException {
verifyBindingsDefined();
@@ -1727,18 +1684,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
- /**
- * Kill a specific container of the cluster
- * @param name cluster name
- * @param args arguments
- * @return exit code
- * @throws YarnException
- * @throws IOException
- */
+ @Override
public int actionKillContainer(String name,
- ActionKillContainerArgs args) throws
- YarnException,
- IOException {
+ ActionKillContainerArgs args) throws YarnException, IOException {
String id = args.id;
if (SliderUtils.isUnset(id)) {
throw new BadCommandArgumentsException("Missing container id");
@@ -1755,14 +1703,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
- /**
- * Echo operation (not currently wired up to command line)
- * @param name cluster name
- * @param args arguments
- * @return the echoed text
- * @throws YarnException
- * @throws IOException
- */
+ @Override
public String actionEcho(String name, ActionEchoArgs args) throws
YarnException,
IOException {
@@ -1790,15 +1731,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws YarnException YARN issues
* @throws IOException IO problems
*/
- private ApplicationReport findInstance(String appname) throws
- YarnException,
- IOException {
+ private ApplicationReport findInstance(String appname)
+ throws YarnException, IOException {
return YarnAppListClient.findInstance(appname);
}
- private RunningApplication findApplication(String appname) throws
- YarnException,
- IOException {
+ private RunningApplication findApplication(String appname)
+ throws YarnException, IOException {
ApplicationReport applicationReport = findInstance(appname);
return applicationReport != null ? new RunningApplication(yarnClient, applicationReport): null;
@@ -1829,9 +1768,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws YarnException
* @throws IOException
*/
- private SliderClusterProtocol connect(ApplicationReport app) throws
- YarnException,
- IOException {
+ private SliderClusterProtocol connect(ApplicationReport app)
+ throws YarnException, IOException {
try {
return RpcBinder.getProxy(getConfig(),
@@ -1846,15 +1784,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
}
- /**
- * Status operation
- *
- * @param clustername cluster name
- * @param statusArgs status arguments
- * @return 0 -for success, else an exception is thrown
- * @throws YarnException
- * @throws IOException
- */
+ @Override
@VisibleForTesting
public int actionStatus(String clustername, ActionStatusArgs statusArgs) throws
YarnException,
@@ -1872,26 +1802,15 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
- /**
- * Version Details
- * @return exit code
- */
+ @Override
public int actionVersion() {
SliderVersionInfo.loadAndPrintVersionInfo(log);
return EXIT_SUCCESS;
}
- /**
- * Stop the cluster
- *
- * @param clustername cluster name
- * @param freezeArgs arguments to the stop
- * @return EXIT_SUCCESS if the cluster was not running by the end of the operation
- */
+ @Override
public int actionFreeze(String clustername,
- ActionFreezeArgs freezeArgs) throws
- YarnException,
- IOException {
+ ActionFreezeArgs freezeArgs) throws YarnException, IOException {
verifyBindingsDefined();
SliderUtils.validateClusterName(clustername);
int waittime = freezeArgs.getWaittime();
@@ -2004,9 +1923,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
@SuppressWarnings(
{"UseOfSystemOutOrSystemErr", "IOResourceOpenedButNotSafelyClosed"})
- public int actionGetConf(String clustername, ActionGetConfArgs confArgs) throws
- YarnException,
- IOException {
+ public int actionGetConf(String clustername, ActionGetConfArgs confArgs)
+ throws YarnException, IOException {
File outfile = null;
if (confArgs.getOutput() != null) {
@@ -2067,9 +1985,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
- /**
- * Restore a cluster
- */
+ @Override
public int actionThaw(String clustername, ActionThawArgs thaw) throws YarnException, IOException {
SliderUtils.validateClusterName(clustername);
// see if it is actually running and bail out;
@@ -2089,10 +2005,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws YarnException
* @throws IOException
*/
- public int flex(String clustername,
- Map<String, Integer> roleInstances) throws
- YarnException,
- IOException {
+ public int flex(String clustername, Map<String, Integer> roleInstances)
+ throws YarnException, IOException {
verifyBindingsDefined();
SliderUtils.validateClusterName(clustername);
Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
@@ -2382,58 +2296,67 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
- /**
- * Registry operation
- *
- * @param args registry Arguments
- * @return 0 for success, -1 for some issues that aren't errors, just failures
- * to retrieve information (e.g. no configurations for that entry)
- * @throws YarnException YARN problems
- * @throws IOException Network or other problems
- */
- public int actionResolve(ActionResolveArgs args) throws
- YarnException,
- IOException {
- // as this is also a test entry point, validate
+ @Override
+ public int actionResolve(ActionResolveArgs args)
+ throws YarnException, IOException {
+ // as this is an API entry point, validate
// the arguments
args.validate();
RegistryOperations operations = getRegistryOperations();
String path = args.path;
Collection<ServiceRecord> serviceRecords;
+ ServiceRecordMarshal serviceRecordMarshal = new ServiceRecordMarshal();
+ File outputPath = args.out;
try {
if (args.list) {
Map<String, ServiceRecord> recordMap =
listServiceRecords(operations, path);
serviceRecords = recordMap.values();
// list records out
+ StringBuilder builder = new StringBuilder(1024);
+ for (Entry<String, ServiceRecord> recordEntry : recordMap
+ .entrySet()) {
+ ServiceRecord instance = recordEntry.getValue();
+ builder.append("\"").append(recordEntry.getKey()).append("\":\n");
+ builder.append(serviceRecordMarshal.toJson(instance));
+ builder.append("}\n");
+ }
+ String records = builder.toString();
+ if (outputPath == null) {
+ print(records);
+ } else {
+ SliderUtils.write(outputPath, records.getBytes("UTF-8"), false);
+ }
} else {
+ // resolve single entry
ServiceRecord instance = resolve(path);
serviceRecords = new ArrayList<ServiceRecord>(1);
serviceRecords.add(instance);
- // list or save records
+ // write out JSON content
+ if (outputPath != null) {
+ byte[] data = serviceRecordMarshal.toBytes(instance);
+ SliderUtils.write(outputPath, data, false);
+ } else {
+ // print to the console
+ print(serviceRecordMarshal.toJson(instance));
+ }
}
// JDK7
- } catch (FileNotFoundException e) {
- log.info("{}", e);
- log.debug("{}", e, e);
- return EXIT_NOT_FOUND;
} catch (PathNotFoundException e) {
- log.info("{}", e);
- log.debug("{}", e, e);
+ // no record at this path
+ return EXIT_NOT_FOUND;
+ } catch (NoRecordException e) {
return EXIT_NOT_FOUND;
+ } catch (InvalidRecordException e) {
+ // it is not a record
+ log.error("{}", e);
+ log.debug("{}", e, e);
+ return EXIT_EXCEPTION_THROWN;
}
return EXIT_SUCCESS;
}
- /**
- * Registry operation
- *
- * @param registryArgs registry Arguments
- * @return 0 for success, -1 for some issues that aren't errors, just failures
- * to retrieve information (e.g. no configurations for that entry)
- * @throws YarnException YARN problems
- * @throws IOException Network or other problems
- */
+ @Override
public int actionRegistry(ActionRegistryArgs registryArgs) throws
YarnException,
IOException {
@@ -2514,22 +2437,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return serviceRecords;
}
- /**
- * diagnostic operation
- *
- * @param clusterName
- * application name
- * @param diagosticArgs
- * diagnostic Arguments
- * @return 0 for success, -1 for some issues that aren't errors, just
- * failures to retrieve information (e.g. no application name
- * specified)
- * @throws YarnException
- * YARN problems
- * @throws IOException
- * Network or other problems
- */
- public int actionDiagnostic(ActionDiagnosticArgs diagnosticArgs) {
+ @Override
+ public int actionDiagnostic(ActionDiagnosticArgs diagnosticArgs) {
try {
if (diagnosticArgs.client) {
actionDiagnosticClient();
@@ -2862,9 +2771,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
/**
- * write out the config
- * @param published
- * @param registryArgs
+ * write out the config. If a destination is provided and that dir is a
+ * directory, the entry is written to it with the name provided + extension,
+ * else it is printed to standard out.
+ * @param published published config
+ * @param registryArgs registry Arguments
* @throws BadCommandArgumentsException
* @throws IOException
*/
@@ -3007,11 +2918,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return registryOperations;
}
- /**
- * Get the registry binding. As this may start the registry, it can take time
- * and fail
- * @return the registry
- */
+ @Override
public RegistryOperations getRegistryOperations()
throws SliderException, IOException {
return maybeStartYarnRegistry();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/032dacf4/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
new file mode 100644
index 0000000..cacf962
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
+import org.apache.slider.common.params.ActionAMSuicideArgs;
+import org.apache.slider.common.params.ActionDiagnosticArgs;
+import org.apache.slider.common.params.ActionEchoArgs;
+import org.apache.slider.common.params.ActionFlexArgs;
+import org.apache.slider.common.params.ActionFreezeArgs;
+import org.apache.slider.common.params.ActionInstallPackageArgs;
+import org.apache.slider.common.params.ActionKillContainerArgs;
+import org.apache.slider.common.params.ActionRegistryArgs;
+import org.apache.slider.common.params.ActionResolveArgs;
+import org.apache.slider.common.params.ActionStatusArgs;
+import org.apache.slider.common.params.ActionThawArgs;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.providers.AbstractClientProvider;
+
+import java.io.IOException;
+
+/**
+ * Interface of those method calls in the slider API that are intended
+ * for direct public invocation.
+ * <p>
+ * Stability: evolving
+ */
+public interface SliderClientAPI extends Service {
+ /**
+ * Destroy a cluster. There's two race conditions here
+ * #1 the cluster is started between verifying that there are no live
+ * clusters of that name.
+ */
+ int actionDestroy(String clustername) throws YarnException,
+ IOException;
+
+ /**
+ * AM to commit an asynchronous suicide
+ */
+ int actionAmSuicide(String clustername,
+ ActionAMSuicideArgs args) throws YarnException, IOException;
+
+ /**
+ * Get the provider for this cluster
+ * @param provider the name of the provider
+ * @return the provider instance
+ * @throws SliderException problems building the provider
+ */
+ AbstractClientProvider createClientProvider(String provider)
+ throws SliderException;
+
+ /**
+ * Build up the cluster specification/directory
+ *
+ * @param clustername cluster name
+ * @param buildInfo the arguments needed to build the cluster
+ * @throws YarnException Yarn problems
+ * @throws IOException other problems
+ * @throws BadCommandArgumentsException bad arguments.
+ */
+ int actionBuild(String clustername,
+ AbstractClusterBuildingActionArgs buildInfo) throws YarnException, IOException;
+
+ /**
+ * Upload application package to user home directory
+ *
+ * @param installPkgInfo the arguments needed to build the cluster
+ * @throws YarnException Yarn problems
+ * @throws IOException other problems
+ * @throws BadCommandArgumentsException bad arguments.
+ */
+ int actionInstallPkg(ActionInstallPackageArgs installPkgInfo)
+ throws YarnException, IOException;
+
+ /**
+ * Update the cluster specification
+ *
+ * @param clustername cluster name
+ * @param buildInfo the arguments needed to update the cluster
+ * @throws YarnException Yarn problems
+ * @throws IOException other problems
+ */
+ int actionUpdate(String clustername,
+ AbstractClusterBuildingActionArgs buildInfo)
+ throws YarnException, IOException;
+ /**
+ * Get the report of a this application
+ * @return the app report or null if it could not be found.
+ * @throws IOException
+ * @throws YarnException
+ */
+ ApplicationReport getApplicationReport()
+ throws IOException, YarnException;
+
+ /**
+ * Kill the submitted application via YARN
+ * @throws YarnException
+ * @throws IOException
+ */
+ boolean forceKillApplication(String reason)
+ throws YarnException, IOException;
+
+ /**
+ * Implement the list action: list all nodes
+ * @return exit code of 0 if a list was created
+ */
+ @VisibleForTesting
+ int actionList(String clustername) throws IOException, YarnException;
+
+ /**
+ * Implement the islive action: probe for a cluster of the given name existing
+ * @return exit code
+ */
+ @VisibleForTesting
+ int actionFlex(String name, ActionFlexArgs args) throws YarnException, IOException;
+
+ /**
+ * Test for a cluster existing probe for a cluster of the given name existing
+ * in the filesystem. If the live param is set, it must be a live cluster
+ * @return exit code
+ */
+ @VisibleForTesting
+ int actionExists(String name, boolean checkLive) throws YarnException, IOException;
+
+ /**
+ * Kill a specific container of the cluster
+ * @param name cluster name
+ * @param args arguments
+ * @return exit code
+ * @throws YarnException
+ * @throws IOException
+ */
+ int actionKillContainer(String name, ActionKillContainerArgs args)
+ throws YarnException, IOException;
+
+ /**
+ * Echo operation (not currently wired up to command line)
+ * @param name cluster name
+ * @param args arguments
+ * @return the echoed text
+ * @throws YarnException
+ * @throws IOException
+ */
+ String actionEcho(String name, ActionEchoArgs args)
+ throws YarnException, IOException;
+
+ /**
+ * Status operation
+ *
+ * @param clustername cluster name
+ * @param statusArgs status arguments
+ * @return 0 -for success, else an exception is thrown
+ * @throws YarnException
+ * @throws IOException
+ */
+ @VisibleForTesting
+ int actionStatus(String clustername, ActionStatusArgs statusArgs)
+ throws YarnException, IOException;
+
+ /**
+ * Version Details
+ * @return exit code
+ */
+ int actionVersion();
+
+ /**
+ * Stop the cluster
+ *
+ * @param clustername cluster name
+ * @param freezeArgs arguments to the stop
+ * @return EXIT_SUCCESS if the cluster was not running by the end of the operation
+ */
+ int actionFreeze(String clustername, ActionFreezeArgs freezeArgs)
+ throws YarnException, IOException;
+
+ /**
+ * Restore a cluster
+ */
+ int actionThaw(String clustername, ActionThawArgs thaw) throws YarnException, IOException;
+
+ /**
+ * Registry operation
+ *
+ * @param args registry Arguments
+ * @return 0 for success, -1 for some issues that aren't errors, just failures
+ * to retrieve information (e.g. no configurations for that entry)
+ * @throws YarnException YARN problems
+ * @throws IOException Network or other problems
+ */
+ int actionResolve(ActionResolveArgs args)
+ throws YarnException, IOException;
+
+ /**
+ * Registry operation
+ *
+ * @param registryArgs registry Arguments
+ * @return 0 for success, -1 for some issues that aren't errors, just failures
+ * to retrieve information (e.g. no configurations for that entry)
+ * @throws YarnException YARN problems
+ * @throws IOException Network or other problems
+ */
+ int actionRegistry(ActionRegistryArgs registryArgs)
+ throws YarnException, IOException;
+
+ /**
+ * diagnostic operation
+ *
+ * @param clusterName
+ * application name
+ * @param diagosticArgs
+ * diagnostic Arguments
+ * @return 0 for success, -1 for some issues that aren't errors, just
+ * failures to retrieve information (e.g. no application name
+ * specified)
+ * @throws YarnException YARN problems
+ * @throws IOException Network or other problems
+ */
+ int actionDiagnostic(ActionDiagnosticArgs diagnosticArgs);
+
+ /**
+ * Get the registry binding. As this may start the registry, it can take time
+ * and fail
+ * @return the registry
+ */
+ RegistryOperations getRegistryOperations()
+ throws SliderException, IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/032dacf4/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
index 9b6a5f1..cf5611c 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
@@ -83,10 +83,5 @@ public class ActionResolveArgs extends AbstractActionArgs {
@Parameter(names = {ARG_VERBOSE},
description = "verbose output")
public boolean verbose;
-
- @Parameter(names = {ARG_INTERNAL},
- description = "fetch internal registry entries")
- public boolean internal;
-
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/032dacf4/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
index 0fffffb..16363d2 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
@@ -191,7 +191,7 @@ public class ClientArgs extends CommonArgs {
* Look at the chosen action and bind it as the core action for the operation.
* In theory this could be done by introspecting on the list of actions and
* choosing it without the switch statement. In practise this switch, while
- * verbose, is easier to debug.
+ * verbose, is easier to debug. And in JDK7, much simpler.
* @throws SliderException bad argument or similar
*/
@Override
@@ -206,7 +206,7 @@ public class ClientArgs extends CommonArgs {
//its a builder, so set those actions too
buildingActionArgs = actionCreateArgs;
- }else if (SliderActions.ACTION_FREEZE.equals(action)) {
+ } else if (SliderActions.ACTION_FREEZE.equals(action)) {
bindCoreAction(actionFreezeArgs);
} else if (SliderActions.ACTION_THAW.equals(action)) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/032dacf4/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 9c93753..72aa4fe 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -69,6 +69,7 @@ import org.slf4j.LoggerFactory;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
@@ -1664,8 +1665,8 @@ public final class SliderUtils {
/**
* Validate an executable
- * @param program
- * @param exe
+ * @param program program name for errors
+ * @param exe program to look at
* @throws IOException
*/
public static void validateExe(String program, File exe) throws IOException {
@@ -1676,6 +1677,29 @@ public final class SliderUtils {
}
}
+ /**
+ * Write bytes to a file
+ * @param outfile output file
+ * @param data data to write
+ * @param createParent flag to indicate that the parent dir should
+ * be created
+ * @throws IOException on any IO problem
+ */
+ public static void write(File outfile, byte[] data, boolean createParent)
+ throws IOException {
+ File parentDir = outfile.getParentFile();
+ if (createParent) {
+ parentDir.mkdirs();
+ }
+ SliderUtils.verifyIsDir(parentDir, log);
+ FileOutputStream out = new FileOutputStream(outfile);
+ try {
+ out.write(data);
+ } finally {
+ IOUtils.closeStream(out);
+ }
+
+ }
/**
* Execute a command for a test operation
@@ -1683,7 +1707,7 @@ public final class SliderUtils {
* @param status status code expected
* @param timeoutMillis timeout in millis for process to finish
* @param logger
- *@param outputString optional string to grep for (must not span a line)
+ * @param outputString optional string to grep for (must not span a line)
* @param commands commands @return the process
* @throws IOException on any failure.
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/032dacf4/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index 1f65d2f..97e995d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -29,6 +29,7 @@ import org.apache.hadoop.registry.client.impl.RegistryOperationsClient
import org.apache.hadoop.registry.client.types.RegistryPathStatus
import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes
+import org.apache.slider.common.params.ActionResolveArgs
import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
import static org.apache.hadoop.registry.client.binding.RegistryUtils.*
@@ -109,9 +110,6 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
ClusterNode master = nodes[0]
assert master.role == SliderKeys.COMPONENT_AM
-
-
-
String username = client.username
def yarnRegistryClient = client.yarnAppListClient
describe("list of all applications")
@@ -174,12 +172,47 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
assert null != serviceRecord.getInternalEndpoint(AGENT_ONEWAY_REST_API)
assert null != serviceRecord.getInternalEndpoint(AGENT_SECURE_REST_API)
+ // use the resolve operation
+ describe "resolve CLI action"
+ ActionResolveArgs resolveArgs = new ActionResolveArgs()
+ resolveArgs.path = recordsPath;
+ resolveArgs.list = true;
+ // to stdout
+ client.actionResolve(resolveArgs)
+ // to a file
+ File destFile = new File("target/resolve.json")
+ destFile.delete()
+ resolveArgs.out = destFile
+ client.actionResolve(resolveArgs)
+ assert destFile.exists()
+ destFile.delete()
+
+ // look at a single record
+ resolveArgs.out = null;
+ resolveArgs.list = false;
+ resolveArgs.path = recordsPath +"/"+ clustername
+ // to stdout
+ client.actionResolve(resolveArgs)
+ resolveArgs.out = destFile
+ client.actionResolve(resolveArgs)
+ assert destFile.exists()
+ ServiceRecordMarshal serviceRecordMarshal = new ServiceRecordMarshal()
+ def recordFromFile = serviceRecordMarshal.fromFile(destFile)
+ assert recordFromFile[YarnRegistryAttributes.YARN_ID] ==
+ serviceRecord[YarnRegistryAttributes.YARN_ID]
+ assert recordFromFile[YarnRegistryAttributes.YARN_PERSISTENCE] ==
+ serviceRecord[YarnRegistryAttributes.YARN_PERSISTENCE]
+
+
+
// hit the registry web page
def registryEndpoint = serviceRecord.getExternalEndpoint(
CustomRegistryConstants.REGISTRY_REST_API)
assert registryEndpoint != null
def registryURL = RegistryTypeUtils.retrieveAddressURLs(registryEndpoint)[0]
+
+ // Look at the Registry WADL
describe("Registry WADL @ $registryURL")
def publisherEndpoint = serviceRecord.getExternalEndpoint(
CustomRegistryConstants.PUBLISHER_REST_API)
@@ -393,6 +426,5 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
assert oldInstance.yarnApplicationState >= YarnApplicationState.FINISHED
-
}
}
[6/8] git commit: Merge branch 'develop' into
feature/SLIDER-149_Support_a_YARN_service_registry
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7f7bb5b8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7f7bb5b8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7f7bb5b8
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 7f7bb5b832a09d3420cc8a3c801b50a5f923e2b5
Parents: bfbc67a 0419a5a
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 7 15:18:04 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 7 15:18:04 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo/appConfig-default.json | 2 +-
app-packages/accumulo/pom.xml | 1 +
pom.xml | 6 +
slider-agent/conf/agent.ini | 1 +
.../src/main/python/agent/AgentConfig.py | 14 ++
.../src/main/python/agent/Controller.py | 35 +++-
.../src/test/python/agent/TestController.py | 63 ++++++
slider-agent/src/test/python/agent/TestMain.py | 37 ++++
.../org/apache/slider/common/SliderKeys.java | 4 +
.../slider/common/tools/CoreFileSystem.java | 15 ++
.../providers/agent/AgentProviderService.java | 27 +++
.../server/appmaster/SliderAppMaster.java | 161 ++++++++++-----
.../security/SecurityConfiguration.java | 201 +++++++++++++++++++
.../security/SecurityConfigurationTest.groovy | 159 +++++++++++++++
.../AppsThroughAgentQueueAndLabelsIT.groovy | 26 +++
15 files changed, 702 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7f7bb5b8/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7f7bb5b8/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7f7bb5b8/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
[7/8] git commit: minor cleanup of utility classes, mostly formatting
Posted by st...@apache.org.
minor cleanup of utility classes, mostly formatting
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c09205f6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c09205f6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c09205f6
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: c09205f66725438854456379664fae8a0fa298ab
Parents: 7f7bb5b
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 7 15:41:14 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 7 15:41:44 2014 -0700
----------------------------------------------------------------------
.../slider/common/tools/CoreFileSystem.java | 31 +++++++-------------
.../apache/slider/common/tools/SliderUtils.java | 14 ++++-----
2 files changed, 17 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c09205f6/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index b6e6ecf..2ea371b 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -158,11 +158,10 @@ public class CoreFileSystem {
* @throws java.io.IOException trouble
* @throws SliderException slider-specific exceptions
*/
- public Path createClusterDirectories(String clustername, Configuration conf) throws
- IOException,
- SliderException {
-
-
+ public Path createClusterDirectories(String clustername, Configuration conf)
+ throws IOException, SliderException {
+
+
Path clusterDirectory = buildClusterDirPath(clustername);
InstancePaths instancePaths = new InstancePaths(clusterDirectory);
createClusterDirectories(instancePaths);
@@ -204,8 +203,8 @@ public class CoreFileSystem {
*
* @param dir directory
* @param clusterPerms cluster permissions
- * @throws IOException IO problem
- * @throws org.apache.slider.core.exceptions.BadClusterStateException any cluster state problem
+ * @throws IOException IO problem
+ * @throws BadClusterStateException any cluster state problem
*/
public void createWithPermissions(Path dir, FsPermission clusterPerms) throws
IOException,
@@ -572,7 +571,7 @@ public class CoreFileSystem {
*
* @param clustername name of the cluster
* @return the path to the spec.
- * @throws IOException IO problems
+ * @throws IOException IO problems
* @throws SliderException if the path isn't there
*/
public Path locateInstanceDefinition(String clustername) throws IOException,
@@ -591,23 +590,15 @@ public class CoreFileSystem {
* @throws IOException IO problems
* @throws SliderException if the cluster specification is not present
*/
- public void verifyClusterSpecExists(String clustername,
- Path clusterSpecPath) throws
- IOException,
+ public void verifyClusterSpecExists(String clustername, Path clusterSpecPath)
+ throws IOException,
SliderException {
if (!fileSystem.isFile(clusterSpecPath)) {
log.debug("Missing specification file {}", clusterSpecPath);
- throw UnknownApplicationInstanceException.unknownInstance(clustername
- +
- "\n (definition not found at "
- +
- clusterSpecPath);
+ throw UnknownApplicationInstanceException.unknownInstance(
+ clustername + "\n (definition not found at " + clusterSpecPath);
}
}
-
- public Path fileToPath(File file) {
- return new Path(file.getAbsoluteFile().toURI());
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c09205f6/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 5313ab9..9c93753 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -511,8 +511,7 @@ public final class SliderUtils {
int length = separator.length();
String s = b.toString();
return (trailing || s.isEmpty()) ?
- s
- : (b.substring(0, b.length() - length));
+ s : (b.substring(0, b.length() - length));
}
/**
@@ -1589,7 +1588,7 @@ public final class SliderUtils {
}
- protected static void verifyIsFile(String program, File exe) throws
+ public static void verifyIsFile(String program, File exe) throws
FileNotFoundException {
if (!exe.isFile()) {
throw new FileNotFoundException(program
@@ -1599,7 +1598,7 @@ public final class SliderUtils {
}
}
- protected static void verifyFileSize(String program,
+ public static void verifyFileSize(String program,
File exe,
int minFileSize) throws FileNotFoundException {
if (exe.length() < minFileSize) {
@@ -1740,7 +1739,6 @@ public final class SliderUtils {
} catch (InterruptedException e) {
throw new InterruptedIOException(e.toString());
} catch (TimeoutException e) {
- log.debug("");
errorText = e.toString();
}
// error text: non null ==> operation failed
@@ -1843,11 +1841,11 @@ public final class SliderUtils {
/**
* validate if a file on HDFS can be open
*
- * @throws IOException
- * : the file can't be found or open
+ * @throws IOException the file can't be found or opened
* @throws URISyntaxException
*/
- public static void validateHDFSFile(SliderFileSystem sliderFileSystem, String pathStr) throws IOException, URISyntaxException{
+ public static void validateHDFSFile(SliderFileSystem sliderFileSystem, String pathStr)
+ throws IOException, URISyntaxException{
URI pathURI = new URI(pathStr);
InputStream inputStream = sliderFileSystem.getFileSystem().open(new Path(pathURI));
if(inputStream == null){
[4/8] git commit: SLIDER-341. Add a window based failure count for
auto-start to limit indefinite attempt
Posted by st...@apache.org.
SLIDER-341. Add a window based failure count for auto-start to limit indefinite attempt
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/961e1704
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/961e1704
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/961e1704
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 961e17043f589c4096dce896d64a582dd000dd4e
Parents: d8b36ca
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Mon Oct 6 20:47:44 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Mon Oct 6 20:47:44 2014 -0700
----------------------------------------------------------------------
slider-agent/conf/agent.ini | 1 +
.../src/main/python/agent/AgentConfig.py | 14 +++++
.../src/main/python/agent/Controller.py | 35 ++++++++++-
.../src/test/python/agent/TestController.py | 63 ++++++++++++++++++++
slider-agent/src/test/python/agent/TestMain.py | 37 ++++++++++++
5 files changed, 148 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/961e1704/slider-agent/conf/agent.ini
----------------------------------------------------------------------
diff --git a/slider-agent/conf/agent.ini b/slider-agent/conf/agent.ini
index 7b9d57d..48113e3 100644
--- a/slider-agent/conf/agent.ini
+++ b/slider-agent/conf/agent.ini
@@ -43,6 +43,7 @@ log_level=INFO
[command]
max_retries=2
sleep_between_retries=1
+auto_restart=5,5
[security]
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/961e1704/slider-agent/src/main/python/agent/AgentConfig.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/AgentConfig.py b/slider-agent/src/main/python/agent/AgentConfig.py
index e45ba23..86925b1 100644
--- a/slider-agent/src/main/python/agent/AgentConfig.py
+++ b/slider-agent/src/main/python/agent/AgentConfig.py
@@ -61,6 +61,7 @@ log_level=INFO
[command]
max_retries=2
sleep_between_retries=1
+auto_restart=5,5
[security]
keysdir=security/keys
@@ -109,6 +110,8 @@ class AgentConfig:
# agent version file
VERSION_FILE = "version_file"
+ AUTO_RESTART = "auto_restart"
+
FOLDER_MAPPING = {
APP_PACKAGE_DIR: "WORK",
APP_INSTALL_DIR: "WORK",
@@ -164,6 +167,17 @@ class AgentConfig:
return ""
return command
+ # return max, window - max failures within window minutes
+ def getErrorWindow(self):
+ window = config.get(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART)
+ if window != None:
+ parts = window.split(',')
+ if len(parts) == 2:
+ if parts[0].isdigit() and parts[1].isdigit():
+ return (int(parts[0]), int(parts[1]))
+ pass
+ return (0, 0)
+
def set(self, category, name, value):
global config
return config.set(category, name, value)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/961e1704/slider-agent/src/main/python/agent/Controller.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Controller.py b/slider-agent/src/main/python/agent/Controller.py
index 11db21c..77f932c 100644
--- a/slider-agent/src/main/python/agent/Controller.py
+++ b/slider-agent/src/main/python/agent/Controller.py
@@ -27,6 +27,7 @@ import time
import threading
import urllib2
import pprint
+import math
from random import randint
from AgentConfig import AgentConfig
@@ -86,7 +87,8 @@ class Controller(threading.Thread):
self.statusCommand = None
self.failureCount = 0
self.heartBeatRetryCount = 0
- self.autoRestart = False
+ self.autoRestartFailures = 0
+ self.autoRestartTrackingSince = 0
def __del__(self):
@@ -275,7 +277,7 @@ class Controller(threading.Thread):
stored_command = self.actionQueue.customServiceOrchestrator.stored_command
if len(stored_command) > 0:
auto_start_command = self.create_start_command(stored_command)
- if auto_start_command:
+ if auto_start_command and self.shouldAutoRestart():
logger.info("Automatically adding a start command.")
logger.debug("Auto start command: " + pprint.pformat(auto_start_command))
self.updateStateBasedOnCommand([auto_start_command], False)
@@ -486,6 +488,35 @@ class Controller(threading.Thread):
return {'exitstatus': 1, 'log': err_msg}
+ # Basic window that only counts failures till the window duration expires
+ def shouldAutoRestart(self):
+ max, window = self.config.getErrorWindow()
+ if max <= 0 or window <= 0:
+ return True
+
+ seconds_now = time.time()
+ if self.autoRestartTrackingSince == 0:
+ self.autoRestartTrackingSince = seconds_now
+ self.autoRestartFailures = 1
+ return True
+
+ self.autoRestartFailures += 1
+ minutes = math.floor((seconds_now - self.autoRestartTrackingSince) / 60)
+ if self.autoRestartFailures > max:
+ logger.info("Auto restart not allowed due to " + str(self.autoRestartFailures) + " failures in " + str(minutes) +
+ " minutes. Max restarts allowed is " + str(max) + " in " + str(window) + " minutes.")
+ return False
+
+ if minutes > window:
+ logger.info("Resetting window as number of minutes passed is " + str(minutes))
+ self.autoRestartTrackingSince = seconds_now
+ self.autoRestartFailures = 1
+ return True
+ return True
+
+ pass
+
+
def main(argv=None):
# Allow Ctrl-C
signal.signal(signal.SIGINT, signal.SIG_DFL)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/961e1704/slider-agent/src/test/python/agent/TestController.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestController.py b/slider-agent/src/test/python/agent/TestController.py
index 401d69a..02b0d0e 100644
--- a/slider-agent/src/test/python/agent/TestController.py
+++ b/slider-agent/src/test/python/agent/TestController.py
@@ -25,6 +25,7 @@ import unittest, threading
from agent import Controller, ActionQueue
from agent import hostname
import sys
+import time
from Controller import AGENT_AUTO_RESTART_EXIT_CODE
from Controller import State
from AgentConfig import AgentConfig
@@ -255,6 +256,68 @@ class TestController(unittest.TestCase):
self.assertTrue(os_exit_mock.call_args[0][0] == AGENT_AUTO_RESTART_EXIT_CODE)
+ @patch("time.time")
+ def test_failure_window(self, mock_time):
+ config = AgentConfig("", "")
+ original_config = config.get(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART)
+ config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, '2,1')
+ ## The behavior of side_effect is different when you run tests in command line and when you do it through IDE
+ ## So few extra items are there in the list
+ mock_time.side_effect = [200, 500, 500]
+ controller5 = Controller.Controller(config)
+
+ try:
+ self.assertTrue(controller5.shouldAutoRestart())
+ self.assertTrue(controller5.shouldAutoRestart())
+ finally:
+ config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, original_config)
+
+
+ @patch("time.time")
+ def test_failure_window(self, mock_time):
+ config = AgentConfig("", "")
+ original_config = config.get(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART)
+ config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, '3,1')
+ ## The behavior of side_effect is different when you run tests in command line and when you do it through IDE
+ ## So few extra items are there in the list
+ mock_time.side_effect = [200, 210, 220, 230, 240, 250]
+ controller5 = Controller.Controller(config)
+
+ try:
+ self.assertTrue(controller5.shouldAutoRestart())
+ self.assertTrue(controller5.shouldAutoRestart())
+ self.assertTrue(controller5.shouldAutoRestart())
+ self.assertFalse(controller5.shouldAutoRestart())
+ finally:
+ config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, original_config)
+
+
+ def test_failure_window2(self):
+ config = MagicMock()
+ config.getErrorWindow.return_value = (0, 0)
+ controller = Controller.Controller(config)
+
+ self.assertTrue(controller.shouldAutoRestart())
+
+ config.getErrorWindow.return_value = (0, 1)
+ self.assertTrue(controller.shouldAutoRestart())
+
+ config.getErrorWindow.return_value = (1, 0)
+ self.assertTrue(controller.shouldAutoRestart())
+
+ config.getErrorWindow.return_value = (-1, -1)
+ self.assertTrue(controller.shouldAutoRestart())
+
+ config.getErrorWindow.return_value = (1, 1)
+ self.assertTrue(controller.shouldAutoRestart())
+
+ #second failure within a minute
+ self.assertFalse(controller.shouldAutoRestart())
+
+ #do not reset unless window expires
+ self.assertFalse(controller.shouldAutoRestart())
+
+
@patch("urllib2.urlopen")
def test_sendRequest(self, requestMock):
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/961e1704/slider-agent/src/test/python/agent/TestMain.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestMain.py b/slider-agent/src/test/python/agent/TestMain.py
index e73a05a..7c0036b 100644
--- a/slider-agent/src/test/python/agent/TestMain.py
+++ b/slider-agent/src/test/python/agent/TestMain.py
@@ -312,6 +312,43 @@ class TestMain(unittest.TestCase):
AgentConfig_set_mock.assert_any_call("server", "zk_reg_path", "/registry/org-apache-slider/cl1")
+ def test_config1(self):
+ config = AgentConfig("", "")
+ (max, window) = config.getErrorWindow()
+ self.assertEqual(max, 5)
+ self.assertEqual(window, 5)
+
+ config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, '')
+ (max, window) = config.getErrorWindow()
+ self.assertEqual(max, 0)
+ self.assertEqual(window, 0)
+
+ config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, '33')
+ (max, window) = config.getErrorWindow()
+ self.assertEqual(max, 0)
+ self.assertEqual(window, 0)
+
+ config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, '-4,-6')
+ (max, window) = config.getErrorWindow()
+ self.assertEqual(max, 0)
+ self.assertEqual(window, 0)
+
+ config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, 'wd,er')
+ (max, window) = config.getErrorWindow()
+ self.assertEqual(max, 0)
+ self.assertEqual(window, 0)
+
+ config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, '2,20')
+ (max, window) = config.getErrorWindow()
+ self.assertEqual(max, 2)
+ self.assertEqual(window, 20)
+
+ config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, ' 2, 30')
+ (max, window) = config.getErrorWindow()
+ self.assertEqual(max, 0)
+ self.assertEqual(window, 0)
+
+
if __name__ == "__main__":
logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG)
unittest.main()
\ No newline at end of file
[3/8] git commit: SLIDER-489 set accumulo app package appConfig java
home in a variable
Posted by st...@apache.org.
SLIDER-489 set accumulo app package appConfig java home in a variable
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d8b36caf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d8b36caf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d8b36caf
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: d8b36cafe8966f9b7cd655e3ead535559403abb6
Parents: 2359c6d
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Mon Oct 6 17:10:37 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Mon Oct 6 17:10:37 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo/appConfig-default.json | 2 +-
app-packages/accumulo/pom.xml | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d8b36caf/app-packages/accumulo/appConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/appConfig-default.json b/app-packages/accumulo/appConfig-default.json
index 62050af..9e04564 100644
--- a/app-packages/accumulo/appConfig-default.json
+++ b/app-packages/accumulo/appConfig-default.json
@@ -4,7 +4,7 @@
},
"global": {
"application.def": ".slider/package/ACCUMULO/${app.package.name}.zip",
- "java_home": "/usr/lib/jvm/java",
+ "java_home": "${app.java.home}",
"site.global.app_user": "${app.user}",
"site.global.app_log_dir": "${AGENT_LOG_ROOT}",
"site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d8b36caf/app-packages/accumulo/pom.xml
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml
index f2f1a35..4cd5cca 100644
--- a/app-packages/accumulo/pom.xml
+++ b/app-packages/accumulo/pom.xml
@@ -41,6 +41,7 @@
<!-- these properties are used in the default and the test appConfigs -->
<hadoop.dir>/usr/lib</hadoop.dir> <!-- hadoop expected to be found at ${hadoop.dir}/hadoop -->
<zk.dir>${hadoop.dir}</zk.dir> <!-- zookeeper expected to be found at ${zk.dir}/zookeeper -->
+ <app.java.home>${java.home}</app.java.home>
<app.user>yarn</app.user>
<app.user.group>hadoop</app.user.group>
<accumulo.keytab></accumulo.keytab>
[2/8] git commit: SLIDER-474 enable keytab-based security for AM
Posted by st...@apache.org.
SLIDER-474 enable keytab-based security for AM
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2359c6dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2359c6dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2359c6dd
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 2359c6ddbaab56c84a4c901b09c025593eee08ef
Parents: 80e8df0
Author: Jon Maron <jm...@hortonworks.com>
Authored: Mon Oct 6 15:50:36 2014 -0400
Committer: Jon Maron <jm...@hortonworks.com>
Committed: Mon Oct 6 15:50:36 2014 -0400
----------------------------------------------------------------------
.../org/apache/slider/common/SliderKeys.java | 4 +
.../slider/common/tools/CoreFileSystem.java | 15 ++
.../providers/agent/AgentProviderService.java | 27 +++
.../server/appmaster/SliderAppMaster.java | 161 ++++++++++-----
.../security/SecurityConfiguration.java | 201 +++++++++++++++++++
.../security/SecurityConfigurationTest.groovy | 159 +++++++++++++++
6 files changed, 520 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2359c6dd/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index ddb9ee0..4348fb0 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -86,6 +86,7 @@ public interface SliderKeys extends SliderXmlConfKeys {
String HISTORY_DIR_NAME = "history";
String HISTORY_FILENAME_SUFFIX = "json";
String HISTORY_FILENAME_PREFIX = "rolehistory-";
+ String KEYTAB_DIR = "keytabs";
/**
* Filename pattern is required to save in strict temporal order.
@@ -170,6 +171,9 @@ public interface SliderKeys extends SliderXmlConfKeys {
String PASSPHRASE = "DEV";
String PASS_LEN = "50";
String KEYSTORE_LOCATION = "ssl.server.keystore.location";
+ String AM_LOGIN_KEYTAB_NAME = "slider.am.login.keytab.name";
+ String AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path";
+ String KEYTAB_PRINCIPAL = "slider.keytab.principal.name";
/**
* Python specific
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2359c6dd/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index 955d991..b6e6ecf 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -134,6 +134,21 @@ public class CoreFileSystem {
}
/**
+ * Build up the path string for keytab install location -no attempt to
+ * create the directory is made
+ *
+ * @return the path for keytab installation location
+ */
+ public Path buildKeytabPath(String keytabName, String applicationName) {
+ Preconditions.checkNotNull(applicationName);
+ Path basePath = getBaseApplicationPath();
+ Path baseKeytabDir = new Path(basePath, SliderKeys.KEYTAB_DIR);
+ Path appKeytabDir = new Path(baseKeytabDir, applicationName);
+ return keytabName == null ? appKeytabDir :
+ new Path(appKeytabDir, keytabName);
+ }
+
+ /**
* Create the Slider cluster path for a named cluster and all its subdirs
* This is a directory; a mkdirs() operation is executed
* to ensure that it is there.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2359c6dd/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 67a268e..88c8709 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -21,6 +21,7 @@ package org.apache.slider.providers.agent;
import com.google.common.annotations.VisibleForTesting;
import org.apache.curator.utils.ZKPaths;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -325,6 +326,32 @@ public class AgentProviderService extends AbstractProviderService implements
launcher.addLocalResource(AgentKeys.AGENT_VERSION_FILE, agentVerRes);
}
+ if (SliderUtils.isHadoopClusterSecure(getConfig())) {
+ String keytabFullPath = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(
+ SliderKeys.AM_KEYTAB_LOCAL_PATH);
+ String amKeytabName = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(
+ SliderKeys.AM_LOGIN_KEYTAB_NAME);
+ if (SliderUtils.isUnset(keytabFullPath)) {
+ // we need to localize the keytab files in the directory
+ Path keytabDir = fileSystem.buildKeytabPath(null,
+ getAmState().getApplicationName());
+ FileStatus[] keytabs = fileSystem.getFileSystem().listStatus(keytabDir);
+ LocalResource keytabRes;
+ for (FileStatus keytab : keytabs) {
+ if (!amKeytabName.equals(keytab.getPath().getName())) {
+ log.info("Localizing keytab {}", keytab.getPath().getName());
+ keytabRes = fileSystem.createAmResource(keytab.getPath(),
+ LocalResourceType.FILE);
+ launcher.addLocalResource(SliderKeys.KEYTAB_DIR + "/" +
+ keytab.getPath().getName(),
+ keytabRes);
+ }
+ }
+ }
+ }
+
//add the configuration resources
launcher.addLocalResources(fileSystem.submitDirectory(
generatedConfPath,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2359c6dd/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 5676f3f..50fc265 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.ipc.ProtocolSignature;
import org.apache.hadoop.security.Credentials;
@@ -121,6 +122,7 @@ import org.apache.slider.server.appmaster.rpc.RpcBinder;
import org.apache.slider.server.appmaster.rpc.SliderAMPolicyProvider;
import org.apache.slider.server.appmaster.rpc.SliderClusterProtocolPBImpl;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.appmaster.security.SecurityConfiguration;
import org.apache.slider.server.appmaster.state.AppState;
import org.apache.slider.server.appmaster.state.ContainerAssignment;
import org.apache.slider.server.appmaster.state.ProviderAppState;
@@ -226,7 +228,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
/**
* token blob
*/
- private ByteBuffer allTokens;
+ private Credentials containerTokens;
private WorkflowRpcService rpcService;
@@ -430,19 +432,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
super.serviceStart();
}
- @Override
- protected void serviceStop() throws Exception {
- super.serviceStop();
-
- if (fsDelegationTokenManager != null) {
- try {
- fsDelegationTokenManager.cancelDelegationToken(getConfig());
- } catch (Exception e) {
- log.info("Error cancelling HDFS delegation token", e);
- }
- }
- }
-
/**
* Start the queue processing
*/
@@ -559,6 +548,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// triggers resolution and snapshotting in agent
appState.updateInstanceDefinition(instanceDefinition);
+
+ Configuration serviceConf = getConfig();
+
File confDir = getLocalConfDir();
if (!confDir.exists() || !confDir.isDirectory()) {
log.info("Conf dir {} does not exist.", confDir);
@@ -566,8 +558,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
log.info("Parent dir {}:\n{}", parentFile, SliderUtils.listDir(parentFile));
}
- Configuration serviceConf = getConfig();
- // IP filtering
+ // IP filtering
serviceConf.set(HADOOP_HTTP_FILTER_INITIALIZERS, AM_FILTER_NAME);
//get our provider
@@ -706,16 +697,49 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// set the RM-defined maximum cluster values
appInformation.put(ResourceKeys.YARN_CORES, Integer.toString(containerMaxCores));
appInformation.put(ResourceKeys.YARN_MEMORY, Integer.toString(containerMaxMemory));
-
- boolean securityEnabled = UserGroupInformation.isSecurityEnabled();
+
+ // process the initial user to obtain the set of user
+ // supplied credentials (tokens were passed in by client). Remove AMRM
+ // token and HDFS delegation token, the latter because we will provide an
+ // up to date token for container launches (getContainerTokens()).
+ UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
+ Credentials credentials = currentUser.getCredentials();
+ Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
+ while (iter.hasNext()) {
+ Token<?> token = iter.next();
+ log.info("Token {}", token.getKind());
+ if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME) ||
+ token.getKind().equals(DelegationTokenIdentifier.HDFS_DELEGATION_KIND)) {
+ iter.remove();
+ }
+ }
+ // at this point this credentials map is probably clear, but leaving this
+ // code to allow for future tokens...
+ containerTokens = credentials;
+
+ SecurityConfiguration securityConfiguration = new SecurityConfiguration(
+ serviceConf, instanceDefinition, clustername);
+ // obtain security state
+ boolean securityEnabled = securityConfiguration.isSecurityEnabled();
if (securityEnabled) {
secretManager.setMasterKey(
amRegistrationData.getClientToAMTokenMasterKey().array());
applicationACLs = amRegistrationData.getApplicationACLs();
- //tell the server what the ACLs are
+ //tell the server what the ACLs are
rpcService.getServer().refreshServiceAcl(serviceConf,
new SliderAMPolicyProvider());
+ // perform keytab based login to establish kerberos authenticated
+ // principal. Can do so now since AM registration with RM above required
+ // tokens associated to principal
+ String principal = securityConfiguration.getPrincipal();
+ File localKeytabFile = securityConfiguration.getKeytabFile(
+ fs, instanceDefinition, principal);
+ // Now log in...
+ login(principal, localKeytabFile);
+ // obtain new FS reference that should be kerberos based and different
+ // than the previously cached reference
+ fs = getClusterFS();
}
// extract container list
@@ -795,27 +819,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
maybeStartMonkey();
// setup token renewal and expiry handling for long lived apps
- if (SliderUtils.isHadoopClusterSecure(getConfig())) {
- fsDelegationTokenManager = new FsDelegationTokenManager(actionQueues);
- fsDelegationTokenManager.acquireDelegationToken(getConfig());
- }
-
- UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
- Credentials credentials =
- currentUser.getCredentials();
- DataOutputBuffer dob = new DataOutputBuffer();
- credentials.writeTokenStorageToStream(dob);
- dob.close();
- // Now remove the AM->RM token so that containers cannot access it.
- Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
- while (iter.hasNext()) {
- Token<?> token = iter.next();
- log.info("Token {}", token.getKind());
- if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
- iter.remove();
- }
- }
- allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+// if (SliderUtils.isHadoopClusterSecure(getConfig())) {
+// fsDelegationTokenManager = new FsDelegationTokenManager(actionQueues);
+// fsDelegationTokenManager.acquireDelegationToken(getConfig());
+// }
// if not a secure cluster, extract the username -it will be
// propagated to workers
@@ -861,6 +868,40 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
return finish();
}
+ protected void login(String principal, File localKeytabFile)
+ throws IOException, SliderException {
+ UserGroupInformation.loginUserFromKeytab(principal,
+ localKeytabFile.getAbsolutePath());
+ validateLoginUser(UserGroupInformation.getLoginUser());
+ }
+
+ /**
+ * Ensure that the user is generated from a keytab and has no HDFS delegation
+ * tokens.
+ *
+ * @param user
+ * @throws SliderException
+ */
+ protected void validateLoginUser(UserGroupInformation user)
+ throws SliderException {
+ if (!user.isFromKeytab()) {
+ throw new SliderException(SliderExitCodes.EXIT_BAD_STATE, "User is "
+ + "not based on a keytab in a secure deployment.");
+ }
+ Credentials credentials =
+ user.getCredentials();
+ Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
+ while (iter.hasNext()) {
+ Token<?> token = iter.next();
+ log.info("Token {}", token.getKind());
+ if (token.getKind().equals(
+ DelegationTokenIdentifier.HDFS_DELEGATION_KIND)) {
+ log.info("Unexpected HDFS delegation token. Removing...");
+ iter.remove();
+ }
+ }
+ }
+
private void startAgentWebApp(MapOperations appInformation,
Configuration serviceConf) throws IOException {
URL[] urls = ((URLClassLoader) AgentWebApp.class.getClassLoader() ).getURLs();
@@ -1387,9 +1428,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
public void onShutdownRequest() {
LOG_YARN.info("Shutdown Request received");
signalAMComplete(new ActionStopSlider("stop",
- EXIT_SUCCESS,
- FinalApplicationStatus.SUCCEEDED,
- "Shutdown requested from RM"));
+ EXIT_SUCCESS,
+ FinalApplicationStatus.SUCCEEDED,
+ "Shutdown requested from RM"));
}
/**
@@ -1558,7 +1599,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
YarnException {
onRpcCall("getNode()");
RoleInstance instance = appState.getLiveInstanceByContainerID(
- request.getUuid());
+ request.getUuid());
return Messages.GetNodeResponseProto.newBuilder()
.setClusterNode(instance.toProtobuf())
.build();
@@ -1571,7 +1612,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
onRpcCall("getClusterNodes()");
List<RoleInstance>
clusterNodes = appState.getLiveInstancesByContainerIDs(
- request.getUuidList());
+ request.getUuidList());
Messages.GetClusterNodesResponseProto.Builder builder =
Messages.GetClusterNodesResponseProto.newBuilder();
@@ -1783,18 +1824,44 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
*/
public void startContainer(Container container,
ContainerLaunchContext ctx,
- RoleInstance instance) {
+ RoleInstance instance) throws IOException {
// Set up tokens for the container too. Today, for normal shell commands,
// the container in distribute-shell doesn't need any tokens. We are
// populating them mainly for NodeManagers to be able to download any
// files in the distributed file-system. The tokens are otherwise also
// useful in cases, for e.g., when one is running a "hadoop dfs" command
// inside the distributed shell.
- ctx.setTokens(allTokens.duplicate());
+
+ // add current HDFS delegation token with an up to date token
+ ByteBuffer tokens = getContainerTokens();
+
+ if (tokens != null) {
+ ctx.setTokens(tokens);
+ } else {
+ log.warn("No delegation tokens obtained and set for launch context");
+ }
appState.containerStartSubmitted(container, instance);
nmClientAsync.startContainerAsync(container, ctx);
}
+ private ByteBuffer getContainerTokens() throws IOException {
+ // a delegation token can be retrieved from filesystem since
+ // the login is via a keytab (see above)
+ ByteBuffer tokens = null;
+ Token hdfsToken = getClusterFS().getFileSystem().getDelegationToken
+ (UserGroupInformation.getLoginUser().getShortUserName());
+ if (hdfsToken != null) {
+ Credentials credentials = new Credentials(containerTokens);
+ credentials.addToken(hdfsToken.getKind(), hdfsToken);
+ DataOutputBuffer dob = new DataOutputBuffer();
+ credentials.writeTokenStorageToStream(dob);
+ dob.close();
+ tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+ }
+
+ return tokens;
+ }
+
@Override // NMClientAsync.CallbackHandler
public void onContainerStopped(ContainerId containerId) {
// do nothing but log: container events from the AM
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2359c6dd/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
new file mode 100644
index 0000000..448d02f
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.security;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ *
+ */
+public class SecurityConfiguration {
+
+ protected static final Logger log =
+ LoggerFactory.getLogger(SecurityConfiguration.class);
+ private final Configuration configuration;
+ private final AggregateConf instanceDefinition;
+ private String clusterName;
+
+ public SecurityConfiguration(Configuration configuration,
+ AggregateConf instanceDefinition,
+ String clusterName) throws SliderException {
+ Preconditions.checkNotNull(configuration);
+ Preconditions.checkNotNull(instanceDefinition);
+ Preconditions.checkNotNull(clusterName);
+ this.configuration = configuration;
+ this.instanceDefinition = instanceDefinition;
+ this.clusterName = clusterName;
+ validate();
+ }
+
+ private void validate() throws SliderException {
+ if (isSecurityEnabled()) {
+ String principal = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(SliderKeys.KEYTAB_PRINCIPAL);
+ if(SliderUtils.isUnset(principal)) {
+ // if no login identity is available, fail
+ UserGroupInformation loginUser = null;
+ try {
+ loginUser = getLoginUser();
+ } catch (IOException e) {
+ throw new SliderException(SliderExitCodes.EXIT_BAD_STATE, e,
+ "No principal configured for the application and"
+ + "exception raised during retrieval of login user. "
+ + "Unable to proceed with application "
+ + "initialization. Please ensure a value "
+ + "for %s exists in the application "
+ + "configuration or the login issue is addressed",
+ SliderKeys.KEYTAB_PRINCIPAL);
+ }
+ if (loginUser == null) {
+ throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ "No principal configured for the application "
+ + "and no login user found. "
+ + "Unable to proceed with application "
+ + "initialization. Please ensure a value "
+ + "for %s exists in the application "
+ + "configuration or the login issue is addressed",
+ SliderKeys.KEYTAB_PRINCIPAL);
+ }
+ }
+ // ensure that either local or distributed keytab mechanism is enabled,
+ // but not both
+ String keytabFullPath = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM)
+ .get(SliderKeys.AM_KEYTAB_LOCAL_PATH);
+ String keytabName = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM)
+ .get(SliderKeys.AM_LOGIN_KEYTAB_NAME);
+ if (SliderUtils.isUnset(keytabFullPath) && SliderUtils.isUnset(keytabName)) {
+ throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ "Either a keytab path on the cluster host (%s) or a"
+ + " keytab to be retrieved from HDFS (%s) are"
+ + " required. Please configure one of the keytab"
+ + " retrieval mechanisms.",
+ SliderKeys.AM_KEYTAB_LOCAL_PATH,
+ SliderKeys.AM_LOGIN_KEYTAB_NAME);
+ }
+ if (SliderUtils.isSet(keytabFullPath) && SliderUtils.isSet(keytabName)) {
+ throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ "Both a keytab on the cluster host (%s) and a"
+ + " keytab to be retrieved from HDFS (%s) are"
+ + " specified. Please configure only one keytab"
+ + " retrieval mechanism.",
+ SliderKeys.AM_KEYTAB_LOCAL_PATH,
+ SliderKeys.AM_LOGIN_KEYTAB_NAME);
+
+ }
+ }
+ }
+
+ protected UserGroupInformation getLoginUser() throws IOException {
+ return UserGroupInformation.getLoginUser();
+ }
+
+ public boolean isSecurityEnabled () {
+ return SliderUtils.isHadoopClusterSecure(configuration);
+ }
+
+ public String getPrincipal () throws IOException {
+ String principal = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(SliderKeys.KEYTAB_PRINCIPAL);
+ if (SliderUtils.isUnset(principal)) {
+ principal = UserGroupInformation.getLoginUser().getShortUserName();
+ log.info("No principal set in the slider configuration. Will use AM login"
+ + " identity {} to attempt keytab-based login", principal);
+ }
+
+ return principal;
+ }
+
+ public File getKeytabFile(SliderFileSystem fs,
+ AggregateConf instanceDefinition, String principal)
+ throws SliderException, IOException {
+ String keytabFullPath = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM)
+ .get(SliderKeys.AM_KEYTAB_LOCAL_PATH);
+ File localKeytabFile;
+ if (SliderUtils.isUnset(keytabFullPath)) {
+ // get the keytab
+ String keytabName = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM).get(SliderKeys.AM_LOGIN_KEYTAB_NAME);
+ log.info("No host keytab file path specified. Downloading keytab {}"
+ + " from HDFS to perform login of using principal {}",
+ keytabName, principal);
+ // download keytab to local, protected directory
+ localKeytabFile = getFileFromFileSystem(fs, keytabName);
+ } else {
+ log.info("Leveraging host keytab file {} to login principal {}",
+ keytabFullPath, principal);
+ localKeytabFile = new File(keytabFullPath);
+ }
+ return localKeytabFile;
+ }
+
+ /**
+ * Download the keytab file from FileSystem to local file.
+ * @param fs
+ * @param keytabName
+ * @return
+ * @throws SliderException
+ * @throws IOException
+ */
+ protected File getFileFromFileSystem(SliderFileSystem fs, String keytabName)
+ throws SliderException, IOException {
+ File keytabDestinationDir = new File(
+ FileUtils.getTempDirectory().getAbsolutePath() +
+ "/keytab" + System.currentTimeMillis());
+ if (!keytabDestinationDir.mkdirs()) {
+ throw new SliderException("Unable to create local keytab directory");
+ }
+ RawLocalFileSystem fileSystem = new RawLocalFileSystem();
+ // allow app user to access local keytab dir
+ FsPermission permissions = new FsPermission(FsAction.ALL, FsAction.NONE,
+ FsAction.NONE);
+ fileSystem.setPermission(new Path(keytabDestinationDir.getAbsolutePath()),
+ permissions);
+
+ Path keytabPath = fs.buildKeytabPath(keytabName, clusterName);
+ File localKeytabFile = new File(keytabDestinationDir, keytabName);
+ FileUtil.copy(fs.getFileSystem(), keytabPath,
+ localKeytabFile,
+ false, configuration);
+ // set permissions on actual keytab file to be read-only for user
+ permissions = new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE);
+ fileSystem.setPermission(new Path(localKeytabFile.getAbsolutePath()),
+ permissions);
+ return localKeytabFile;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2359c6dd/slider-core/src/test/groovy/org/apache/slider/server/appmaster/security/SecurityConfigurationTest.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/security/SecurityConfigurationTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/security/SecurityConfigurationTest.groovy
new file mode 100644
index 0000000..1dcbd9c
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/security/SecurityConfigurationTest.groovy
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.security
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic
+import org.apache.hadoop.security.UserGroupInformation
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.core.conf.AggregateConf
+import org.apache.slider.core.conf.MapOperations
+import org.apache.slider.core.exceptions.SliderException;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class SecurityConfigurationTest {
+ final shouldFail = new GroovyTestCase().&shouldFail
+
+ @Test
+ public void testValidLocalConfiguration() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderKeys.KEYTAB_PRINCIPAL, "test")
+ compOps.put(SliderKeys.AM_KEYTAB_LOCAL_PATH, "/some/local/path")
+
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster")
+ }
+
+ @Test
+ public void testValidDistributedConfiguration() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderKeys.KEYTAB_PRINCIPAL, "test")
+ compOps.put(SliderKeys.AM_LOGIN_KEYTAB_NAME, "some.keytab")
+
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster")
+ }
+
+ @Test
+ public void testMissingPrincipalNoLoginWithDistributedConfig() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderKeys.AM_LOGIN_KEYTAB_NAME, "some.keytab")
+
+ shouldFail(SliderException) {
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster") {
+ @Override
+ protected UserGroupInformation getLoginUser() throws IOException {
+ return null
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testMissingPrincipalNoLoginWithLocalConfig() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderKeys.AM_KEYTAB_LOCAL_PATH, "/some/local/path")
+
+ shouldFail(SliderException) {
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster") {
+ @Override
+ protected UserGroupInformation getLoginUser() throws IOException {
+ return null
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testBothKeytabMechanismsConfigured() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderKeys.KEYTAB_PRINCIPAL, "test")
+ compOps.put(SliderKeys.AM_KEYTAB_LOCAL_PATH, "/some/local/path")
+ compOps.put(SliderKeys.AM_LOGIN_KEYTAB_NAME, "some.keytab")
+
+ shouldFail(SliderException) {
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster")
+ }
+ }
+
+ @Test
+ public void testNoKeytabMechanismConfigured() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderKeys.KEYTAB_PRINCIPAL, "test")
+
+ shouldFail(SliderException) {
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster")
+ }
+ }
+
+ @Test
+ public void testMissingPrincipalButLoginWithDistributedConfig() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderKeys.AM_LOGIN_KEYTAB_NAME, "some.keytab")
+
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster")
+ }
+
+ @Test
+ public void testMissingPrincipalButLoginWithLocalConfig() throws Throwable {
+ Configuration config = new Configuration()
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos")
+ AggregateConf aggregateConf = new AggregateConf();
+ MapOperations compOps =
+ aggregateConf.appConfOperations.getOrAddComponent(SliderKeys.COMPONENT_AM)
+ compOps.put(SliderKeys.AM_KEYTAB_LOCAL_PATH, "/some/local/path")
+
+ SecurityConfiguration securityConfiguration =
+ new SecurityConfiguration(config, aggregateConf, "testCluster")
+ }
+}