You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/11/04 22:37:04 UTC
[1/5] incubator-slider git commit: SLIDER-82 minor test source cleanup
Repository: incubator-slider
Updated Branches:
refs/heads/develop c8fb17e05 -> 227339e5c
refs/heads/feature/SLIDER-82-pass-3 4e9a95b92 -> dd6073352
SLIDER-82 minor test source cleanup
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/da24357a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/da24357a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/da24357a
Branch: refs/heads/feature/SLIDER-82-pass-3
Commit: da24357a4de4cdf1c494eaf830441fe49bf48355
Parents: 4e9a95b
Author: Steve Loughran <st...@apache.org>
Authored: Wed Nov 4 18:09:58 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Nov 4 18:10:07 2015 +0000
----------------------------------------------------------------------
.../model/appstate/TestMockAppStateAppRestIntegration.groovy | 5 -----
.../model/appstate/TestMockAppStateDynamicRoles.groovy | 6 ------
.../appmaster/model/appstate/TestMockAppStateFlexing.groovy | 3 +--
.../model/appstate/TestMockAppStateRMOperations.groovy | 4 +---
.../model/appstate/TestMockAppStateRebuildOnAMRestart.groovy | 3 +--
.../model/appstate/TestMockContainerResourceAllocations.groovy | 5 -----
6 files changed, 3 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da24357a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
index d36fdbc..6739623 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
@@ -41,11 +41,6 @@ import org.junit.Test
@Slf4j
class TestMockAppStateAppRestIntegration extends BaseMockAppStateTest implements MockRoles {
- @Override
- String getTestName() {
- return "TestMockAppStateAppRestIntegration"
- }
-
@Test
public void testCachedIntDocument() throws Throwable {
ContentCache cache = new ContentCache()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da24357a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
index e35f028..05b38ab 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
@@ -20,23 +20,17 @@ package org.apache.slider.server.appmaster.model.appstate
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.slider.api.ResourceKeys
import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
-import org.apache.slider.server.appmaster.state.AppState
-import org.apache.slider.server.appmaster.state.AppStateBindingInfo
import org.apache.slider.server.appmaster.state.ContainerPriority
import org.apache.slider.server.appmaster.state.RoleHistoryUtils
import org.apache.slider.server.appmaster.state.RoleInstance
-import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
import org.junit.Test
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da24357a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
index 257092a..548842c 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
@@ -20,7 +20,6 @@ package org.apache.slider.server.appmaster.model.appstate
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Container
-import org.apache.slider.api.ClusterDescription
import org.apache.slider.core.exceptions.TriggerClusterTeardownException
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockRoles
@@ -76,7 +75,7 @@ class TestMockAppStateFlexing extends BaseMockAppStateTest implements MockRoles
ops = appState.reviewRequestAndReleaseNodes()
assert ops.empty
- RoleInstance ri2 = appState.innerOnNodeManagerContainerStarted(target.id)
+ appState.innerOnNodeManagerContainerStarted(target.id)
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da24357a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
index 9ac6fcf..ba7588a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
@@ -110,7 +110,6 @@ class TestMockAppStateRMOperations extends BaseMockAppStateTest implements MockR
// four outstanding
assert role0.requested == 4
-
// flex cluster to 3
role0.desired = 3
ops = appState.reviewRequestAndReleaseNodes()
@@ -226,8 +225,7 @@ class TestMockAppStateRMOperations extends BaseMockAppStateTest implements MockR
public void testFlexUpNoSpace() throws Throwable {
// engine only has two nodes, so > 2 will be outstanding
engine = new MockYarnEngine(1, 2)
- List<AbstractRMOperation> ops
- // role: desired = 2, requested = 1, actual=1
+ // role: desired = 2, requested = 1, actual=1
def role0 = role0Status
role0.desired = 4
createAndSubmitNodes()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da24357a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
index 02052c2..59cc2c8 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
@@ -36,8 +36,7 @@ import org.junit.Test
*/
@CompileStatic
@Slf4j
-class TestMockAppStateRebuildOnAMRestart extends BaseMockAppStateTest
- implements MockRoles {
+class TestMockAppStateRebuildOnAMRestart extends BaseMockAppStateTest implements MockRoles {
@Override
String getTestName() {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/da24357a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
index ad607cf..93cce95 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
@@ -38,11 +38,6 @@ import org.junit.Test
@Slf4j
class TestMockContainerResourceAllocations extends BaseMockAppStateTest {
- @Override
- String getTestName() {
- "TestMockContainerResourceAllocations"
- }
-
@Test
public void testNormalAllocations() throws Throwable {
ConfTree clusterSpec = factory.newConfTree(1, 0, 0)
[2/5] incubator-slider git commit: SLIDER-82 setting up node listings
into AppState binding
Posted by st...@apache.org.
SLIDER-82 setting up node listings into AppState binding
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c6231fe7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c6231fe7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c6231fe7
Branch: refs/heads/feature/SLIDER-82-pass-3
Commit: c6231fe769190d6da21c77e6c74298b08f49319b
Parents: da24357
Author: Steve Loughran <st...@apache.org>
Authored: Wed Nov 4 18:11:28 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Nov 4 18:11:28 2015 +0000
----------------------------------------------------------------------
.../slider/client/SliderYarnClientImpl.java | 18 +++++++++--
.../server/appmaster/SliderAppMaster.java | 34 ++++++++++++--------
.../appmaster/state/AppStateBindingInfo.java | 3 ++
.../slider/agent/rest/TestStandaloneREST.groovy | 4 +--
.../appmaster/model/mock/MockYarnCluster.groovy | 11 +++----
5 files changed, 44 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c6231fe7/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index 42759fd..803ccd6 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -20,6 +20,7 @@ package org.apache.slider.client;
import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
@@ -29,6 +30,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
@@ -42,6 +44,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.net.BindException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
@@ -64,6 +67,17 @@ public class SliderYarnClientImpl extends YarnClientImpl {
*/
public static final String KILL_ALL = "all";
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ String addr = conf.get(YarnConfiguration.RM_ADDRESS);
+ if (addr.startsWith("0.0.0.0")) {
+ // address isn't known; fail fast
+ throw new BindException("Invalid " + YarnConfiguration.RM_ADDRESS + " value:" + addr
+ + " - see https://wiki.apache.org/hadoop/UnsetHostnameOrPort");
+ }
+ super.serviceInit(conf);
+ }
+
/**
* Get the RM Client RPC interface
* @return an RPC interface valid after initialization and authentication
@@ -107,7 +121,6 @@ public class SliderYarnClientImpl extends YarnClientImpl {
return results;
}
-
/**
* find all instances of a specific app -if there is more than one in the
* YARN cluster,
@@ -141,8 +154,7 @@ public class SliderYarnClientImpl extends YarnClientImpl {
public boolean isApplicationLive(ApplicationReport app) {
Preconditions.checkArgument(app != null, "Null app report");
- return app.getYarnApplicationState().ordinal() <=
- YarnApplicationState.RUNNING.ordinal();
+ return app.getYarnApplicationState().ordinal() <= YarnApplicationState.RUNNING.ordinal();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c6231fe7/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index b552290..e6a5bd5 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -57,9 +57,9 @@ import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
@@ -68,7 +68,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
@@ -173,6 +172,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.net.BindException;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URL;
@@ -215,8 +215,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
*/
protected static final Logger LOG_YARN = log;
- public static final String SERVICE_CLASSNAME_SHORT =
- "SliderAppMaster";
+ public static final String SERVICE_CLASSNAME_SHORT = "SliderAppMaster";
public static final String SERVICE_CLASSNAME =
"org.apache.slider.server.appmaster." + SERVICE_CLASSNAME_SHORT;
@@ -495,8 +494,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
metrics.registerAll(new GarbageCollectorMetricSet());
*/
- contentCache = ApplicationResouceContentCacheFactory.createContentCache(
- stateForProviders);
+ contentCache = ApplicationResouceContentCacheFactory.createContentCache(stateForProviders);
executorService = new WorkflowExecutorService<>("AmExecutor",
Executors.newFixedThreadPool(2,
@@ -504,6 +502,14 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
addService(executorService);
addService(actionQueues);
+ // set up the YARN client. This may require patching in the RM client-API address if it
+ // is (somehow) unset server-side.
+ String clientRMaddr = conf.get(YarnConfiguration.RM_ADDRESS);
+ if (clientRMaddr.startsWith("0.0.0.0")) {
+ // address isn't known; fail fast
+ throw new BindException("Invalid " + YarnConfiguration.RM_ADDRESS + " value:" + addr
+ + " - see https://wiki.apache.org/hadoop/UnsetHostnameOrPort");
+ }
addService(yarnClient = new SliderYarnClientImpl());
//init all child services
@@ -564,8 +570,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//dump the system properties if in debug mode
if (log.isDebugEnabled()) {
- log.debug("System properties:\n" +
- SliderUtils.propertiesToString(System.getProperties()));
+ log.debug("System properties:\n" + SliderUtils.propertiesToString(System.getProperties()));
}
//choose the action
@@ -634,8 +639,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// obtain security state
securityEnabled = securityConfiguration.isSecurityEnabled();
// set the global security flag for the instance definition
- instanceDefinition.getAppConfOperations().set(
- KEY_SECURITY_ENABLED, securityEnabled);
+ instanceDefinition.getAppConfOperations().set(KEY_SECURITY_ENABLED, securityEnabled);
// triggers resolution and snapshotting for agent
appState.setInitialInstanceDefinition(instanceDefinition);
@@ -653,8 +657,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
InternalKeys.INTERNAL_PROVIDER_NAME);
log.info("Cluster provider type is {}", providerType);
SliderProviderFactory factory =
- SliderProviderFactory.createSliderProviderFactory(
- providerType);
+ SliderProviderFactory.createSliderProviderFactory(providerType);
providerService = factory.createServerProvider();
// init the provider BUT DO NOT START IT YET
initAndAddService(providerService);
@@ -673,8 +676,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* turned into an (incompete) container
*/
appMasterContainerID = ConverterUtils.toContainerId(
- SliderUtils.mandatoryEnvVariable(
- ApplicationConstants.Environment.CONTAINER_ID.name()));
+ SliderUtils.mandatoryEnvVariable(ApplicationConstants.Environment.CONTAINER_ID.name()));
appAttemptID = appMasterContainerID.getApplicationAttemptId();
ApplicationId appid = appAttemptID.getApplicationId();
@@ -687,6 +689,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
Map<String, String> envVars;
List<Container> liveContainers;
+ List<NodeReport> nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
+ log.info("Yarn node report count: {}", nodeReports.size());
+
/*
* It is critical this section is synchronized, to stop async AM events
* arriving while registering a restarting AM.
@@ -844,6 +849,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
binding.liveContainers = liveContainers;
binding.applicationInfo = appInformation;
binding.releaseSelector = providerService.createContainerReleaseSelector();
+ binding.nodeReports = nodeReports;
appState.buildInstance(binding);
providerService.rebuildContainerDetails(liveContainers,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c6231fe7/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
index 184c8aa..a2a0b60 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.providers.ProviderRole;
@@ -46,6 +47,8 @@ public class AppStateBindingInfo {
public List<Container> liveContainers = new ArrayList<>(0);
public Map<String, String> applicationInfo = new HashMap<>();
public ContainerReleaseSelector releaseSelector = new SimpleReleaseSelector();
+ /** node reports off the RM. If null, the app state needs to be given a node update later */
+ public List<NodeReport> nodeReports = new ArrayList<>(0);
public void validate() throws IllegalArgumentException {
Preconditions.checkArgument(instanceDefinition != null, "null instanceDefinition");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c6231fe7/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
index 97b3009..29fa51a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
@@ -41,7 +41,6 @@ import static org.apache.slider.server.appmaster.web.rest.RestPaths.*
@Slf4j
class TestStandaloneREST extends AgentMiniClusterTestBase {
-
@Test
public void testStandaloneREST() throws Throwable {
@@ -65,8 +64,7 @@ class TestStandaloneREST extends AgentMiniClusterTestBase {
ApplicationReport report = waitForClusterLive(client)
def proxyAM = report.trackingUrl
def directAM = report.originalTrackingUrl
-
-
+
// set up url config to match
initHttpTestSupport(launcher.configuration)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c6231fe7/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
index 6056e3a..99a9213 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
@@ -147,14 +147,15 @@ public class MockYarnCluster {
/**
* Model cluster nodes on the simpler "slot" model than the YARN-era
- * resource allocation model. Why? Makes it easier to implement.
+ * resource allocation model. Why? Easier to implement scheduling.
+ * Of course, if someone does want to implement the full process...
*
- * When a cluster is offline,
*/
public static class MockYarnClusterNode {
public final int nodeIndex
public final String hostname;
+ public List<String> labels = []
public final MockNodeId nodeId;
public final MockYarnClusterContainer[] containers;
private boolean offline;
@@ -230,8 +231,6 @@ public class MockYarnCluster {
}
return result
}
-
-
/**
* Release a container
@@ -291,8 +290,8 @@ public class MockYarnCluster {
return (hostIndex << 8) | containerIndex & 0xff;
}
- public static final int extractHost(int cid) {
- return (cid >>> 8);
+ public static final int extractHost(long cid) {
+ return (cid >>> 8) & 0xffff;
}
public static final int extractContainer(int cid) {
[3/5] incubator-slider git commit: SLIDER-82 trying to set up
YarnClient,
having config propagation issues in minicluster (RM client addr is 0.0.0.0)
Posted by st...@apache.org.
SLIDER-82 trying to set up YarnClient, having config propagation issues in minicluster (RM client addr is 0.0.0.0)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/dd607335
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/dd607335
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/dd607335
Branch: refs/heads/feature/SLIDER-82-pass-3
Commit: dd6073352d7c7b8690098fee19ecef5b9d4b321f
Parents: c6231fe
Author: Steve Loughran <st...@apache.org>
Authored: Wed Nov 4 21:12:53 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Nov 4 21:12:53 2015 +0000
----------------------------------------------------------------------
.../slider/client/SliderYarnClientImpl.java | 8 +++---
.../slider/core/launch/ContainerLauncher.java | 12 +++------
.../server/appmaster/SliderAppMaster.java | 27 ++++++++++----------
3 files changed, 23 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dd607335/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index 803ccd6..867603b 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -45,6 +45,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.BindException;
+import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
@@ -69,10 +70,11 @@ public class SliderYarnClientImpl extends YarnClientImpl {
@Override
protected void serviceInit(Configuration conf) throws Exception {
- String addr = conf.get(YarnConfiguration.RM_ADDRESS);
- if (addr.startsWith("0.0.0.0")) {
+ InetSocketAddress clientRpcAddress = SliderUtils.getRmAddress(conf);
+ if (!SliderUtils.isAddressDefined(clientRpcAddress)) {
// address isn't known; fail fast
- throw new BindException("Invalid " + YarnConfiguration.RM_ADDRESS + " value:" + addr
+ throw new BindException("Invalid " + YarnConfiguration.RM_ADDRESS
+ + " value:" + conf.get(YarnConfiguration.RM_ADDRESS)
+ " - see https://wiki.apache.org/hadoop/UnsetHostnameOrPort");
}
super.serviceInit(conf);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dd607335/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
index f8ea4ee..69b937d 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
@@ -55,17 +55,13 @@ public class ContainerLauncher extends AbstractLauncher {
public UserGroupInformation setupUGI() {
UserGroupInformation user =
UserGroupInformation.createRemoteUser(container.getId().toString());
- String cmIpPortStr =
- container.getNodeId().getHost() + ":" + container.getNodeId().getPort();
- final InetSocketAddress cmAddress =
- NetUtils.createSocketAddr(cmIpPortStr);
+ String cmIpPortStr = container.getNodeId().getHost() + ":" + container.getNodeId().getPort();
+ final InetSocketAddress cmAddress = NetUtils.createSocketAddr(cmIpPortStr);
- org.apache.hadoop.yarn.api.records.Token containerToken =
- container.getContainerToken();
+ org.apache.hadoop.yarn.api.records.Token containerToken = container.getContainerToken();
if (containerToken != null) {
Token<ContainerTokenIdentifier> token =
- ConverterUtils.convertFromYarn(containerToken,
- cmAddress);
+ ConverterUtils.convertFromYarn(containerToken, cmAddress);
user.addToken(token);
}
return user;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dd607335/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index e6a5bd5..1a127cf 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -502,15 +502,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
addService(executorService);
addService(actionQueues);
- // set up the YARN client. This may require patching in the RM client-API address if it
- // is (somehow) unset server-side.
- String clientRMaddr = conf.get(YarnConfiguration.RM_ADDRESS);
- if (clientRMaddr.startsWith("0.0.0.0")) {
- // address isn't known; fail fast
- throw new BindException("Invalid " + YarnConfiguration.RM_ADDRESS + " value:" + addr
- + " - see https://wiki.apache.org/hadoop/UnsetHostnameOrPort");
- }
- addService(yarnClient = new SliderYarnClientImpl());
//init all child services
super.serviceInit(conf);
@@ -542,8 +533,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* @param args argument list
*/
@Override // RunService
- public Configuration bindArgs(Configuration config, String... args) throws
- Exception {
+ public Configuration bindArgs(Configuration config, String... args) throws Exception {
// let the superclass process it
Configuration superConf = super.bindArgs(config, args);
// add the slider XML config
@@ -667,10 +657,21 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
sliderAMProvider = new SliderAMProviderService();
initAndAddService(sliderAMProvider);
- InetSocketAddress address = SliderUtils.getRmSchedulerAddress(serviceConf);
- log.info("RM is at {}", address);
+ InetSocketAddress rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(serviceConf);
+ log.info("RM is at {}", rmSchedulerAddress);
yarnRPC = YarnRPC.create(serviceConf);
+ // set up the YARN client. This may require patching in the RM client-API address if it
+ // is (somehow) unset server-side. String clientRMaddr = serviceConf.get(YarnConfiguration.RM_ADDRESS);
+ InetSocketAddress clientRpcAddress = SliderUtils.getRmAddress(serviceConf);
+ if (!SliderUtils.isAddressDefined(clientRpcAddress)) {
+ // client addr is being unset. We can lift it from the other RM APIs
+ serviceConf.set(YarnConfiguration.RM_ADDRESS,
+ String.format("%s:%d", rmSchedulerAddress.getHostString(), clientRpcAddress.getPort() ));
+ }
+ initAndAddService(yarnClient = new SliderYarnClientImpl());
+ yarnClient.start();
+
/*
* Extract the container ID. This is then
* turned into an (incompete) container
[4/5] incubator-slider git commit: SLIDER-960 switch to groovy 2.4.4
(the ASF edition)
Posted by st...@apache.org.
SLIDER-960 switch to groovy 2.4.4 (the ASF edition)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/88f881b5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/88f881b5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/88f881b5
Branch: refs/heads/develop
Commit: 88f881b56ebd52bcbf102465a20939ff09253dc8
Parents: c8fb17e
Author: Steve Loughran <st...@apache.org>
Authored: Wed Nov 4 21:14:56 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Nov 4 21:14:56 2015 +0000
----------------------------------------------------------------------
pom.xml | 8 +-------
1 file changed, 1 insertion(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/88f881b5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2a7e41b..3943ea9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -113,7 +113,7 @@
-->
<project.java.src.version>1.7</project.java.src.version>
<enforced.java.version>${project.java.src.version}</enforced.java.version>
- <groovy.version>2.4.0</groovy.version>
+ <groovy.version>2.4.5</groovy.version>
<!--
test options
@@ -603,12 +603,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
<version>${hadoop.version}</version>
<exclusions>
[5/5] incubator-slider git commit: SLIDER-961 clean up SliderClient
code
Posted by st...@apache.org.
SLIDER-961 clean up SliderClient code
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/227339e5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/227339e5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/227339e5
Branch: refs/heads/develop
Commit: 227339e5c661571a989aa3f5e2d7ff5ff9ac667d
Parents: 88f881b
Author: Steve Loughran <st...@apache.org>
Authored: Wed Nov 4 21:36:47 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Nov 4 21:36:47 2015 +0000
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 717 +++++++------------
.../slider/common/tools/CoreFileSystem.java | 2 +-
.../apache/slider/common/tools/SliderUtils.java | 25 +-
.../slider/core/launch/AppMasterLauncher.java | 5 +-
.../core/launch/ClasspathConstructor.java | 1 -
.../slider/core/launch/CommandLineBuilder.java | 55 ++
.../utility/AbstractSliderLaunchedService.java | 20 +-
.../agent/actions/TestActionPackage.groovy | 7 +-
8 files changed, 333 insertions(+), 499 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index de0e07e..07c915c 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -19,7 +19,6 @@
package org.apache.slider.client;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
import com.google.common.io.Files;
import org.apache.commons.collections.CollectionUtils;
@@ -65,9 +64,6 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
-import org.apache.slider.api.InternalKeys;
-import org.apache.slider.api.OptionKeys;
-import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.SliderClusterProtocol;
import org.apache.slider.api.StateValues;
import org.apache.slider.api.proto.Messages;
@@ -77,7 +73,6 @@ import org.apache.slider.client.ipc.SliderClusterOperations;
import org.apache.slider.common.Constants;
import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.params.AbstractActionArgs;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
import org.apache.slider.common.params.ActionAMSuicideArgs;
@@ -106,11 +101,9 @@ import org.apache.slider.common.params.Arguments;
import org.apache.slider.common.params.ClientArgs;
import org.apache.slider.common.params.CommonArgs;
import org.apache.slider.common.params.LaunchArgsAccessor;
-import org.apache.slider.common.params.SliderActions;
import org.apache.slider.common.tools.ConfigHelper;
import org.apache.slider.common.tools.Duration;
import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.common.tools.SliderVersionInfo;
import org.apache.slider.core.build.InstanceBuilder;
import org.apache.slider.core.build.InstanceIO;
@@ -132,7 +125,6 @@ import org.apache.slider.core.exceptions.UsageException;
import org.apache.slider.core.exceptions.WaitTimeoutException;
import org.apache.slider.core.launch.AppMasterLauncher;
import org.apache.slider.core.launch.ClasspathConstructor;
-import org.apache.slider.core.launch.CommandLineBuilder;
import org.apache.slider.core.launch.JavaCommandLineBuilder;
import org.apache.slider.core.launch.LaunchedApplication;
import org.apache.slider.core.launch.RunningApplication;
@@ -203,38 +195,13 @@ import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.ServiceRecordMarshal;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.currentUser;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.extractServiceRecords;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.listServiceRecords;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.serviceclassPath;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.statChildren;
-import static org.apache.slider.common.params.SliderActions.ACTION_AM_SUICIDE;
-import static org.apache.slider.common.params.SliderActions.ACTION_BUILD;
-import static org.apache.slider.common.params.SliderActions.ACTION_CLIENT;
-import static org.apache.slider.common.params.SliderActions.ACTION_CREATE;
-import static org.apache.slider.common.params.SliderActions.ACTION_DEPENDENCY;
-import static org.apache.slider.common.params.SliderActions.ACTION_DESTROY;
-import static org.apache.slider.common.params.SliderActions.ACTION_DIAGNOSTICS;
-import static org.apache.slider.common.params.SliderActions.ACTION_EXISTS;
-import static org.apache.slider.common.params.SliderActions.ACTION_FLEX;
-import static org.apache.slider.common.params.SliderActions.ACTION_FREEZE;
-import static org.apache.slider.common.params.SliderActions.ACTION_HELP;
-import static org.apache.slider.common.params.SliderActions.ACTION_INSTALL_KEYTAB;
-import static org.apache.slider.common.params.SliderActions.ACTION_INSTALL_PACKAGE;
-import static org.apache.slider.common.params.SliderActions.ACTION_KEYTAB;
-import static org.apache.slider.common.params.SliderActions.ACTION_KILL_CONTAINER;
-import static org.apache.slider.common.params.SliderActions.ACTION_LIST;
-import static org.apache.slider.common.params.SliderActions.ACTION_LOOKUP;
-import static org.apache.slider.common.params.SliderActions.ACTION_PACKAGE;
-import static org.apache.slider.common.params.SliderActions.ACTION_REGISTRY;
-import static org.apache.slider.common.params.SliderActions.ACTION_RESOLVE;
-import static org.apache.slider.common.params.SliderActions.ACTION_STATUS;
-import static org.apache.slider.common.params.SliderActions.ACTION_THAW;
-import static org.apache.slider.common.params.SliderActions.ACTION_UPDATE;
-import static org.apache.slider.common.params.SliderActions.ACTION_UPGRADE;
-import static org.apache.slider.common.params.SliderActions.ACTION_VERSION;
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.*;
+import static org.apache.slider.api.InternalKeys.*;
+import static org.apache.slider.api.OptionKeys.*;
+import static org.apache.slider.api.ResourceKeys.*;
+import static org.apache.slider.common.params.SliderActions.*;
+import static org.apache.slider.common.tools.SliderUtils.*;
+
/**
* Client service for Slider
@@ -287,6 +254,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
/**
* The YARN registry service
*/
+ @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
private RegistryOperations registryOperations;
/**
@@ -317,19 +285,19 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
ConfigHelper.injectSliderXMLResource();
// yarn-ify
YarnConfiguration yarnConfiguration = new YarnConfiguration(config);
- return SliderUtils.patchConfiguration(yarnConfiguration);
+ return patchConfiguration(yarnConfiguration);
}
@Override
protected void serviceInit(Configuration conf) throws Exception {
- Configuration clientConf = SliderUtils.loadSliderClientXML();
+ Configuration clientConf = loadSliderClientXML();
ConfigHelper.mergeConfigurations(conf, clientConf, SLIDER_CLIENT_XML, true);
serviceArgs.applyDefinitions(conf);
serviceArgs.applyFileSystemBinding(conf);
// init security with our conf
- if (SliderUtils.isHadoopClusterSecure(conf)) {
- SliderUtils.forceLogin();
- SliderUtils.initProcessSecurity(conf);
+ if (isHadoopClusterSecure(conf)) {
+ forceLogin();
+ initProcessSecurity(conf);
}
AbstractActionArgs coreAction = serviceArgs.getCoreAction();
if (coreAction.getHadoopServicesRequired()) {
@@ -362,9 +330,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// choose the action
String action = serviceArgs.getAction();
- if (SliderUtils.isUnset(action)) {
- throw new SliderException(EXIT_USAGE,
- serviceArgs.usage());
+ if (isUnset(action)) {
+ throw new SliderException(EXIT_USAGE, serviceArgs.usage());
}
int exitCode = EXIT_SUCCESS;
@@ -483,7 +450,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return exitCode;
}
-/**
+ /**
* Perform everything needed to init the hadoop binding.
* This assumes that the service is already in inited or started state
* @throws IOException
@@ -491,7 +458,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
*/
protected void initHadoopBinding() throws IOException, SliderException {
// validate the client
- SliderUtils.validateSliderClientEnvironment(null);
+ validateSliderClientEnvironment(null);
//create the YARN client
yarnClient = new SliderYarnClientImpl();
yarnClient.init(getConfig());
@@ -626,7 +593,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
public int actionDestroy(String clustername,
ActionDestroyArgs destroyArgs) throws YarnException, IOException {
// verify that a live cluster isn't there
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
//no=op, it is now mandatory.
verifyBindingsDefined();
verifyNoLiveClusters(clustername, "Destroy");
@@ -645,9 +612,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
throw new UsageException("Destroy will permanently delete directories and registries. "
+ "Reissue this command with the --force option if you want to proceed.");
}
- boolean deleted =
- fs.delete(clusterDirectory, true);
- if (!deleted) {
+ if (!fs.delete(clusterDirectory, true)) {
log.warn("Filesystem returned false from delete() operation");
}
@@ -743,33 +708,26 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// Otherwise the internal app config and resources states of the app will be
// unwantedly modified and the change will take effect to the running app
// immediately.
- if (template != null && resources == null) {
- throw new BadCommandArgumentsException(
+ require(!(template != null && resources == null),
"Option %s must be specified with option %s",
Arguments.ARG_RESOURCES, Arguments.ARG_TEMPLATE);
- }
- if (resources != null && template == null) {
- throw new BadCommandArgumentsException(
+
+ require(!(resources != null && template == null),
"Option %s must be specified with option %s",
Arguments.ARG_TEMPLATE, Arguments.ARG_RESOURCES);
- }
// For upgrade spec, both --template and --resources should be specified
// and neither of --containers or --components should be used
if (template != null && resources != null) {
- if (CollectionUtils.isNotEmpty(containers)) {
- throw new BadCommandArgumentsException(
+ require(CollectionUtils.isEmpty(containers),
"Option %s cannot be specified with %s or %s",
Arguments.ARG_CONTAINERS, Arguments.ARG_TEMPLATE,
Arguments.ARG_RESOURCES);
- }
- if (CollectionUtils.isNotEmpty(components)) {
- throw new BadCommandArgumentsException(
- "Option %s cannot be specified with %s or %s",
- Arguments.ARG_COMPONENTS, Arguments.ARG_TEMPLATE,
- Arguments.ARG_RESOURCES);
- }
-
+ require(CollectionUtils.isEmpty(components),
+ "Option %s cannot be specified with %s or %s",
+ Arguments.ARG_COMPONENTS, Arguments.ARG_TEMPLATE,
+ Arguments.ARG_RESOURCES);
+
// not an error to try to upgrade a stopped cluster, just return success
// code, appropriate log messages have already been dumped
if (!isAppInRunningState(clustername)) {
@@ -794,7 +752,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private int actionUpgradeContainers(String clustername,
ActionUpgradeArgs upgradeArgs) throws YarnException, IOException {
verifyBindingsDefined();
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
int waittime = upgradeArgs.getWaittime(); // ignored for now
String text = "Upgrade containers";
log.debug("actionUpgradeContainers({}, reason={}, wait={})", clustername,
@@ -892,13 +850,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return false;
}
log.debug("App to upgrade was found: {}:\n{}", clustername,
- new SliderUtils.OnDemandReportStringifier(app));
- if (app.getYarnApplicationState().ordinal() >= YarnApplicationState.FINISHED
- .ordinal()) {
- log.info(
- "Cluster {} is in a terminated state {}. Use command '{}' instead.",
- clustername, app.getYarnApplicationState(),
- SliderActions.ACTION_UPDATE);
+ new OnDemandReportStringifier(app));
+ if (app.getYarnApplicationState().ordinal() >= YarnApplicationState.FINISHED.ordinal()) {
+ log.info("Cluster {} is in a terminated state {}. Use command '{}' instead.",
+ clustername, app.getYarnApplicationState(), ACTION_UPDATE);
return false;
}
@@ -915,7 +870,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private static void checkForCredentials(Configuration conf,
ConfTree tree) throws IOException {
- if (tree.credentials == null || tree.credentials.size()==0) {
+ if (tree.credentials == null || tree.credentials.isEmpty()) {
log.info("No credentials requested");
return;
}
@@ -930,10 +885,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
Configuration c = new Configuration(conf);
c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider);
- CredentialProvider credentialProvider =
- CredentialProviderFactory.getProviders(c).get(0);
- Set<String> existingAliases =
- new HashSet<>(credentialProvider.getAliases());
+ CredentialProvider credentialProvider = CredentialProviderFactory.getProviders(c).get(0);
+ Set<String> existingAliases = new HashSet<>(credentialProvider.getAliases());
for (String alias : aliases) {
if (existingAliases.contains(alias.toLowerCase(Locale.ENGLISH))) {
log.info("Credentials for " + alias + " found in " + provider);
@@ -949,21 +902,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
}
} finally {
- if (br != null) {
- br.close();
- }
+ org.apache.hadoop.io.IOUtils.closeStream(br);
}
}
private static char[] readOnePassword(String alias) throws IOException {
- BufferedReader br = null;
- try {
- br = new BufferedReader(new InputStreamReader(System.in));
+ try(BufferedReader br = new BufferedReader(new InputStreamReader(System.in))) {
return readPassword(alias, br);
- } finally {
- if (br != null) {
- br.close();
- }
}
}
@@ -1049,12 +994,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
Path fileInFs = new Path(pkgPath, keytabInfo.keytab );
log.info("Deleting keytab {}", fileInFs);
- if (!sliderFileSystem.getFileSystem().exists(fileInFs)) {
- throw new BadCommandArgumentsException("No keytab to delete found at " +
- fileInFs.toUri().toString());
- }
-
- sliderFileSystem.getFileSystem().delete(fileInFs, false);
+ FileSystem sfs = sliderFileSystem.getFileSystem();
+ require(sfs.exists(fileInFs), "No keytab to delete found at %s", fileInFs.toUri());
+ sfs.delete(fileInFs, false);
return EXIT_SUCCESS;
}
@@ -1062,40 +1004,31 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private int actionInstallKeytab(ActionKeytabArgs keytabInfo)
throws BadCommandArgumentsException, IOException {
Path srcFile = null;
- if (StringUtils.isEmpty(keytabInfo.folder)) {
- throw new BadCommandArgumentsException(
- "A valid destination keytab sub-folder name is required (e.g. 'security').\n"
- + CommonArgs.usage(serviceArgs, ACTION_KEYTAB));
- }
+ require(isSet(keytabInfo.folder),
+ "A valid destination keytab sub-folder name is required (e.g. 'security').\n"
+ + CommonArgs.usage(serviceArgs, ACTION_KEYTAB));
- if (StringUtils.isEmpty(keytabInfo.keytab)) {
- throw new BadCommandArgumentsException("A valid local keytab location is required.");
- } else {
- File keytabFile = new File(keytabInfo.keytab);
- if (!keytabFile.exists() || keytabFile.isDirectory()) {
- throw new BadCommandArgumentsException("Unable to access supplied keytab file at " +
- keytabFile.getAbsolutePath());
- } else {
- srcFile = new Path(keytabFile.toURI());
- }
- }
+ requireArgumentSet(Arguments.ARG_KEYTAB, keytabInfo.keytab);
+ File keytabFile = new File(keytabInfo.keytab);
+ require(keytabFile.isFile(),
+ "Unable to access supplied keytab file at %s", keytabFile.getAbsolutePath());
+ srcFile = new Path(keytabFile.toURI());
Path pkgPath = sliderFileSystem.buildKeytabInstallationDirPath(keytabInfo.folder);
- sliderFileSystem.getFileSystem().mkdirs(pkgPath);
- sliderFileSystem.getFileSystem().setPermission(pkgPath, new FsPermission(
+ FileSystem sfs = sliderFileSystem.getFileSystem();
+ sfs.mkdirs(pkgPath);
+ sfs.setPermission(pkgPath, new FsPermission(
FsAction.ALL, FsAction.NONE, FsAction.NONE));
Path fileInFs = new Path(pkgPath, srcFile.getName());
- log.info("Installing keytab {} at {} and overwrite is {}.", srcFile, fileInFs, keytabInfo.overwrite);
- if (sliderFileSystem.getFileSystem().exists(fileInFs) && !keytabInfo.overwrite) {
- throw new BadCommandArgumentsException("Keytab exists at " +
- fileInFs.toUri().toString() +
- ". Use --overwrite to overwrite.");
- }
+ log.info("Installing keytab {} at {} and overwrite is {}.",
+ srcFile, fileInFs, keytabInfo.overwrite);
+ require(!(sfs.exists(fileInFs) && !keytabInfo.overwrite),
+ "Keytab exists at %s. Use --overwrite to overwrite.", fileInFs.toUri());
- sliderFileSystem.getFileSystem().copyFromLocalFile(false, keytabInfo.overwrite, srcFile, fileInFs);
- sliderFileSystem.getFileSystem().setPermission(fileInFs, new FsPermission(
- FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE));
+ sfs.copyFromLocalFile(false, keytabInfo.overwrite, srcFile, fileInFs);
+ sfs.setPermission(fileInFs,
+ new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE));
return EXIT_SUCCESS;
}
@@ -1111,44 +1044,29 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
public int actionInstallPkg(ActionInstallPackageArgs installPkgInfo) throws
YarnException,
IOException {
- log.warn("The " + SliderActions.ACTION_INSTALL_PACKAGE
+ log.warn("The " + ACTION_INSTALL_PACKAGE
+ " option has been deprecated. Please use '"
- + SliderActions.ACTION_PACKAGE + " " + ClientArgs.ARG_INSTALL + "'.");
- Path srcFile = null;
+ + ACTION_PACKAGE + " " + ClientArgs.ARG_INSTALL + "'.");
if (StringUtils.isEmpty(installPkgInfo.name)) {
throw new BadCommandArgumentsException(
- E_INVALID_APPLICATION_TYPE_NAME +"\n"
+ E_INVALID_APPLICATION_TYPE_NAME + "\n"
+ CommonArgs.usage(serviceArgs, ACTION_INSTALL_PACKAGE));
}
-
- if (StringUtils.isEmpty(installPkgInfo.packageURI)) {
- throw new BadCommandArgumentsException(E_INVALID_APPLICATION_PACKAGE_LOCATION);
- } else {
- File pkgFile = new File(installPkgInfo.packageURI);
- if (!pkgFile.exists() || pkgFile.isDirectory()) {
- throw new BadCommandArgumentsException(
- E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE +": "
- + pkgFile.getAbsolutePath());
- } else {
- srcFile = new Path(pkgFile.toURI());
- }
- }
+ Path srcFile = extractPackagePath(installPkgInfo.packageURI);
// Do not provide new options to install-package command as it is in
// deprecated mode. So version is kept null here. Use package --install.
Path pkgPath = sliderFileSystem.buildPackageDirPath(installPkgInfo.name,
null);
- sliderFileSystem.getFileSystem().mkdirs(pkgPath);
+ FileSystem sfs = sliderFileSystem.getFileSystem();
+ sfs.mkdirs(pkgPath);
Path fileInFs = new Path(pkgPath, srcFile.getName());
- log.info("Installing package {} at {} and overwrite is {}.", srcFile, fileInFs, installPkgInfo.replacePkg);
- if (sliderFileSystem.getFileSystem().exists(fileInFs) && !installPkgInfo.replacePkg) {
- throw new BadCommandArgumentsException(
- "Package exists at " + fileInFs.toUri().toString() +"."
- + E_USE_REPLACEPKG_TO_OVERWRITE);
- }
-
- sliderFileSystem.getFileSystem().copyFromLocalFile(false, installPkgInfo.replacePkg, srcFile, fileInFs);
+ log.info("Installing package {} at {} and overwrite is {}.",
+ srcFile, fileInFs, installPkgInfo.replacePkg);
+ require(!(sfs.exists(fileInFs) && !installPkgInfo.replacePkg),
+ "Package exists at %s. : %s", fileInFs.toUri(), E_USE_REPLACEPKG_TO_OVERWRITE);
+ sfs.copyFromLocalFile(false, installPkgInfo.replacePkg, srcFile, fileInFs);
return EXIT_SUCCESS;
}
@@ -1177,11 +1095,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
+ CommonArgs.usage(serviceArgs, ACTION_CLIENT));
}
- if (clientInfo.name == null) {
- throw new BadCommandArgumentsException("No application name specified\n"
- + CommonArgs.usage(serviceArgs,
- ACTION_CLIENT));
- }
+ requireArgumentSet(Arguments.ARG_NAME, clientInfo.name);
File storeFile = null;
SecurityStore.StoreType type;
@@ -1193,11 +1107,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
type = SecurityStore.StoreType.truststore;
}
- if (storeFile.exists()) {
- throw new BadCommandArgumentsException("File %s already exists. "
- + "Please remove that file or select a different file name.",
- storeFile.getAbsolutePath());
- }
+ require (!storeFile.exists(),
+ "File %s already exists. Please remove that file or select a different file name.",
+ storeFile.getAbsolutePath());
String hostname = null;
if (type == SecurityStore.StoreType.keystore) {
hostname = clientInfo.hostname;
@@ -1232,10 +1144,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
}
- byte[]
- keystore =
- createClusterOperations(clientInfo.name).getClientCertificateStore(
- hostname, "client", password, type.name());
+ byte[] keystore = createClusterOperations(clientInfo.name)
+ .getClientCertificateStore(hostname, "client", password, type.name());
// persist to file
IOUtils.write(keystore, new FileOutputStream(storeFile));
@@ -1245,31 +1155,20 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private int doClientInstall(ActionClientArgs clientInfo)
throws IOException, SliderException {
- if (clientInfo.installLocation == null) {
- throw new BadCommandArgumentsException(
+ require(clientInfo.installLocation != null,
E_INVALID_INSTALL_LOCATION +"\n"
+ CommonArgs.usage(serviceArgs, ACTION_CLIENT));
- } else {
- if (!clientInfo.installLocation.exists()) {
- throw new BadCommandArgumentsException(E_INSTALL_PATH_DOES_NOT_EXIST
- +": " + clientInfo.installLocation.getAbsolutePath());
- }
- if (!clientInfo.installLocation.isDirectory()) {
- throw new BadCommandArgumentsException(E_INVALID_INSTALL_PATH
- +": " + clientInfo.installLocation.getAbsolutePath());
- }
- }
+ require(clientInfo.installLocation.exists(),
+ E_INSTALL_PATH_DOES_NOT_EXIST + ": " + clientInfo.installLocation.getAbsolutePath());
+
+ require(clientInfo.installLocation.isFile(),
+ E_INVALID_INSTALL_PATH + ": " + clientInfo.installLocation.getAbsolutePath());
File pkgFile;
- if (StringUtils.isEmpty(clientInfo.packageURI)) {
- throw new BadCommandArgumentsException(E_INVALID_APPLICATION_PACKAGE_LOCATION);
- } else {
- pkgFile = new File(clientInfo.packageURI);
- if (!pkgFile.exists() || pkgFile.isDirectory()) {
- throw new BadCommandArgumentsException(E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE
- +" at " + pkgFile.getAbsolutePath());
- }
- }
+ requireArgumentSet(Arguments.ARG_PACKAGE, clientInfo.packageURI);
+ pkgFile = new File(clientInfo.packageURI);
+ require(pkgFile.isFile(),
+ E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE + " at %s", pkgFile.getAbsolutePath());
JSONObject config = null;
if(clientInfo.clientConfig != null) {
@@ -1368,7 +1267,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
Path appDefPath = null;
try {
appDefPath = new Path(
- SliderUtils.getApplicationDefinitionPath(instanceDefinition
+ getApplicationDefinitionPath(instanceDefinition
.getAppConfOperations()));
} catch (BadConfigException e) {
// Invalid cluster state, so move on to next. No need to log anything
@@ -1390,11 +1289,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
println("%-25s %15s %30s %s", clusterName, packageName,
packageVersion, appDefPathStr);
}
- } catch(IOException e) {
- if(log.isDebugEnabled()) {
- log.debug(clusterName + " application definition path "
- + appDefPathStr + " is not found.");
- }
+ } catch (IOException e) {
+ log.debug("{} application definition path {} is not found.", clusterName, appDefPathStr);
}
}
return EXIT_SUCCESS;
@@ -1403,19 +1299,19 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private int actionPackageList() throws IOException {
Path pkgPath = sliderFileSystem.buildPackageDirPath(StringUtils.EMPTY,
StringUtils.EMPTY);
- log.info("Package install path : " + pkgPath);
- if (!sliderFileSystem.getFileSystem().isDirectory(pkgPath)) {
+ log.info("Package install path : {}", pkgPath);
+ FileSystem sfs = sliderFileSystem.getFileSystem();
+ if (!sfs.isDirectory(pkgPath)) {
log.info("No package(s) installed");
return EXIT_SUCCESS;
}
- FileStatus[] fileStatus = sliderFileSystem.getFileSystem().listStatus(
- pkgPath);
+ FileStatus[] fileStatus = sfs.listStatus(pkgPath);
boolean hasPackage = false;
StringBuilder sb = new StringBuilder();
sb.append("List of installed packages:\n");
for (FileStatus fstat : fileStatus) {
if (fstat.isDirectory()) {
- sb.append("\t" + fstat.getPath().getName());
+ sb.append("\t").append(fstat.getPath().getName());
sb.append("\n");
hasPackage = true;
}
@@ -1428,53 +1324,31 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
- private int actionPackageInstall(ActionPackageArgs actionPackageArgs) throws
- YarnException,
- IOException {
-
- Path srcFile = null;
- if (StringUtils.isEmpty(actionPackageArgs.name)) {
- throw new BadCommandArgumentsException(
- "A valid application type name is required (e.g. HBASE).\n"
- + CommonArgs.usage(serviceArgs, ACTION_PACKAGE));
- }
+ private int actionPackageInstall(ActionPackageArgs actionPackageArgs)
+ throws YarnException, IOException {
+ requireArgumentSet(Arguments.ARG_NAME, actionPackageArgs.name);
- if (StringUtils.isEmpty(actionPackageArgs.packageURI)) {
- throw new BadCommandArgumentsException(
- E_INVALID_APPLICATION_PACKAGE_LOCATION);
- } else {
- File pkgFile = new File(actionPackageArgs.packageURI);
- if (!pkgFile.exists() || pkgFile.isDirectory()) {
- throw new BadCommandArgumentsException(
- E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE
- + ": " + pkgFile.getAbsolutePath());
- } else {
- srcFile = new Path(pkgFile.toURI());
- }
- }
+ Path srcFile = extractPackagePath(actionPackageArgs.packageURI);
Path pkgPath = sliderFileSystem.buildPackageDirPath(actionPackageArgs.name,
actionPackageArgs.version);
- if (!sliderFileSystem.getFileSystem().exists(pkgPath)) {
- sliderFileSystem.getFileSystem().mkdirs(pkgPath);
+ FileSystem fs = sliderFileSystem.getFileSystem();
+ if (!fs.exists(pkgPath)) {
+ fs.mkdirs(pkgPath);
}
Path fileInFs = new Path(pkgPath, srcFile.getName());
- if (sliderFileSystem.getFileSystem().exists(fileInFs)
- && !actionPackageArgs.replacePkg) {
- throw new BadCommandArgumentsException(E_PACKAGE_EXISTS +" at " +
- fileInFs.toUri() + ". Use --replacepkg to overwrite.");
- }
+ require(actionPackageArgs.replacePkg || !fs.exists(fileInFs),
+ E_PACKAGE_EXISTS +" at %s. Use --replacepkg to overwrite.", fileInFs.toUri());
log.info("Installing package {} to {} (overwrite set to {})", srcFile,
fileInFs, actionPackageArgs.replacePkg);
- sliderFileSystem.getFileSystem().copyFromLocalFile(false,
- actionPackageArgs.replacePkg, srcFile, fileInFs);
+ fs.copyFromLocalFile(false, actionPackageArgs.replacePkg, srcFile, fileInFs);
String destPathWithHomeDir = Path
.getPathWithoutSchemeAndAuthority(fileInFs).toString();
String destHomeDir = Path.getPathWithoutSchemeAndAuthority(
- sliderFileSystem.getFileSystem().getHomeDirectory()).toString();
+ fs.getHomeDirectory()).toString();
// a somewhat contrived approach to stripping out the home directory and any trailing
// separator; designed to work on windows and unix
String destPathWithoutHomeDir;
@@ -1492,23 +1366,26 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
+ private Path extractPackagePath(String packageURI)
+ throws BadCommandArgumentsException {
+ require(isSet(packageURI), E_INVALID_APPLICATION_PACKAGE_LOCATION);
+ File pkgFile = new File(packageURI);
+ require(pkgFile.isFile(),
+ E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE + ": " + pkgFile.getAbsolutePath());
+ return new Path(pkgFile.toURI());
+ }
+
private int actionPackageDelete(ActionPackageArgs actionPackageArgs) throws
YarnException, IOException {
- if (StringUtils.isEmpty(actionPackageArgs.name)) {
- throw new BadCommandArgumentsException(
- "A valid application type name is required (e.g. HBASE).\n"
- + CommonArgs.usage(serviceArgs, ACTION_PACKAGE));
- }
+ requireArgumentSet(Arguments.ARG_NAME, actionPackageArgs.name);
Path pkgPath = sliderFileSystem.buildPackageDirPath(actionPackageArgs.name,
actionPackageArgs.version);
- if (!sliderFileSystem.getFileSystem().exists(pkgPath)) {
- throw new BadCommandArgumentsException(E_PACKAGE_DOES_NOT_EXIST +": "
- + pkgPath.toUri().toString());
- }
+ FileSystem fs = sliderFileSystem.getFileSystem();
+ require(fs.exists(pkgPath), E_PACKAGE_DOES_NOT_EXIST +": %s ", pkgPath.toUri());
log.info("Deleting package {} at {}.", actionPackageArgs.name, pkgPath);
- if(sliderFileSystem.getFileSystem().delete(pkgPath, true)) {
+ if(fs.delete(pkgPath, true)) {
log.info("Deleted package {} " + actionPackageArgs.name);
return EXIT_SUCCESS;
} else {
@@ -1548,7 +1425,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
boolean liveClusterAllowed, boolean isUpgradeFlow) throws YarnException,
IOException {
// verify that a live cluster isn't there
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
verifyBindingsDefined();
if (!liveClusterAllowed) {
verifyNoLiveClusters(clustername, "Create");
@@ -1627,8 +1504,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
String count = roleEntry.getValue();
String key = roleEntry.getKey();
log.info("{} => {}", key, count);
- resources.getOrAddComponent(key)
- .put(ResourceKeys.COMPONENT_INSTANCES, count);
+ resources.getOrAddComponent(key).put(COMPONENT_INSTANCES, count);
}
//all CLI role options
@@ -1663,7 +1539,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
builder.setQueue(buildInfo.queue);
String quorum = buildInfo.getZKhosts();
- if (SliderUtils.isUnset(quorum)) {
+ if (isUnset(quorum)) {
quorum = registryQuorum;
}
if (isUnset(quorum)) {
@@ -1735,7 +1611,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
if (!SliderKeys.COMPONENT_AM.equals(componentName)) {
clientComponentInstances.put(componentName, clientResources
.getComponentOptInt(componentName,
- ResourceKeys.COMPONENT_INSTANCES, -1));
+ COMPONENT_INSTANCES, -1));
}
}
@@ -1745,8 +1621,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
} catch (LockAcquireFailedException e) {
log.warn("Failed to get a Lock on cluster resource : {}", e, e);
throw new BadClusterStateException(
- "Failed to load client resource definition " + clustername + ": "
- + e);
+ "Failed to load client resource definition " + clustername + ": " + e, e);
}
Map<String, Integer> clusterComponentInstances = new HashMap<>();
for (Map.Entry<String, Map<String, String>> component : clusterConf
@@ -1755,7 +1630,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
clusterComponentInstances.put(
component.getKey(),
Integer.decode(component.getValue().get(
- ResourceKeys.COMPONENT_INSTANCES)));
+ COMPONENT_INSTANCES)));
}
}
@@ -1763,16 +1638,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
Iterator<Map.Entry<String, Integer>> clientComponentInstanceIt = clientComponentInstances
.entrySet().iterator();
while (clientComponentInstanceIt.hasNext()) {
- Map.Entry<String, Integer> clientComponentInstanceEntry = clientComponentInstanceIt
- .next();
- if (clusterComponentInstances
- .containsKey(clientComponentInstanceEntry.getKey())) {
+ Map.Entry<String, Integer> clientComponentInstanceEntry = clientComponentInstanceIt.next();
+ if (clusterComponentInstances.containsKey(clientComponentInstanceEntry.getKey())) {
// compare instance count now and remove from both maps if they match
if (clusterComponentInstances
.get(clientComponentInstanceEntry.getKey()) == clientComponentInstanceEntry
.getValue()) {
- clusterComponentInstances
- .remove(clientComponentInstanceEntry.getKey());
+ clusterComponentInstances.remove(clientComponentInstanceEntry.getKey());
clientComponentInstanceIt.remove();
}
}
@@ -1846,8 +1718,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
public FsPermission getClusterDirectoryPermissions(Configuration conf) {
String clusterDirPermsOct =
- conf.get(CLUSTER_DIRECTORY_PERMISSIONS,
- DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS);
+ conf.get(CLUSTER_DIRECTORY_PERMISSIONS, DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS);
return new FsPermission(clusterDirPermsOct);
}
@@ -1857,9 +1728,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws BadCommandArgumentsException the exception raised on an invalid config
*/
public void verifyBindingsDefined() throws BadCommandArgumentsException {
- InetSocketAddress rmAddr = SliderUtils.getRmAddress(getConfig());
+ InetSocketAddress rmAddr = getRmAddress(getConfig());
if (!getConfig().getBoolean(YarnConfiguration.RM_HA_ENABLED, false)
- && !SliderUtils.isAddressDefined(rmAddr)) {
+ && !isAddressDefined(rmAddr)) {
throw new BadCommandArgumentsException(
E_NO_RESOURCE_MANAGER
+ " in the argument "
@@ -1923,9 +1794,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws UnknownApplicationInstanceException if the file is not found
*/
public AggregateConf loadInstanceDefinitionUnresolved(String name,
- Path clusterDirectory) throws
- IOException,
- SliderException {
+ Path clusterDirectory) throws IOException, SliderException {
try {
AggregateConf definition =
@@ -1937,7 +1806,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
throw UnknownApplicationInstanceException.unknownInstance(name, e);
}
}
- /**
+
+ /**
* Load the instance definition.
* @param name cluster name
* @param resolved flag to indicate the cluster should be resolved
@@ -1980,11 +1850,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
deployedClusterName = clustername;
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
verifyNoLiveClusters(clustername, "Launch");
Configuration config = getConfig();
lookupZKQuorum();
- boolean clusterSecure = SliderUtils.isHadoopClusterSecure(config);
+ boolean clusterSecure = isHadoopClusterSecure(config);
//create the Slider AM provider -this helps set up the AM
SliderAMClientProvider sliderAM = new SliderAMClientProvider(config);
@@ -1997,18 +1867,15 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
ConfTreeOperations appOperations = instanceDefinition.getAppConfOperations();
Path generatedConfDirPath =
createPathThatMustExist(internalOptions.getMandatoryOption(
- InternalKeys.INTERNAL_GENERATED_CONF_PATH));
+ INTERNAL_GENERATED_CONF_PATH));
Path snapshotConfPath =
createPathThatMustExist(internalOptions.getMandatoryOption(
- InternalKeys.INTERNAL_SNAPSHOT_CONF_PATH));
+ INTERNAL_SNAPSHOT_CONF_PATH));
// cluster Provider
AbstractClientProvider provider = createClientProvider(
- internalOptions.getMandatoryOption(
- InternalKeys.INTERNAL_PROVIDER_NAME));
- // make sure the conf dir is valid;
-
+ internalOptions.getMandatoryOption(INTERNAL_PROVIDER_NAME));
if (log.isDebugEnabled()) {
log.debug(instanceDefinition.toString());
}
@@ -2018,7 +1885,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// add the tags if available
Set<String> applicationTags = provider.getApplicationTags(sliderFileSystem,
- SliderUtils.getApplicationDefinitionPath(appOperations));
+ getApplicationDefinitionPath(appOperations));
AppMasterLauncher amLauncher = new AppMasterLauncher(clustername,
SliderKeys.APP_TYPE,
config,
@@ -2043,8 +1910,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
String libdir = "lib";
Path libPath = new Path(tempPath, libdir);
sliderFileSystem.getFileSystem().mkdirs(libPath);
- log.debug("FS={}, tempPath={}, libdir={}",
- sliderFileSystem, tempPath, libPath);
+ log.debug("FS={}, tempPath={}, libdir={}", sliderFileSystem, tempPath, libPath);
// set local resources for the application master
// local files or archives as needed
@@ -2055,9 +1921,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
boolean hasServerLog4jProperties = false;
Path remoteConfPath = null;
String relativeConfDir = null;
- String confdirProp =
- System.getProperty(SliderKeys.PROPERTY_CONF_DIR);
- if (confdirProp == null || confdirProp.isEmpty()) {
+ String confdirProp = System.getProperty(SliderKeys.PROPERTY_CONF_DIR);
+ if (isUnset(confdirProp)) {
log.debug("No local configuration directory provided as system property");
} else {
File confDir = new File(confdirProp);
@@ -2065,11 +1930,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
throw new BadConfigException(E_CONFIGURATION_DIRECTORY_NOT_FOUND,
confDir);
}
- Path localConfDirPath = SliderUtils.createLocalPath(confDir);
+ Path localConfDirPath = createLocalPath(confDir);
remoteConfPath = new Path(clusterDirectory, SliderKeys.SUBMITTED_CONF_DIR);
log.debug("Slider configuration directory is {}; remote to be {}",
localConfDirPath, remoteConfPath);
- SliderUtils.copyDirectory(config, localConfDirPath, remoteConfPath, null);
+ copyDirectory(config, localConfDirPath, remoteConfPath, null);
File log4jserver =
new File(confDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME);
@@ -2089,7 +1954,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
Map<String, LocalResource> submittedConfDir =
sliderFileSystem.submitDirectory(remoteConfPath,
relativeConfDir);
- SliderUtils.mergeMaps(localResources, submittedConfDir);
+ mergeMaps(localResources, submittedConfDir);
}
}
// build up the configuration
@@ -2109,7 +1974,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
Configuration clientConfExtras = new Configuration(false);
// then build up the generated path.
FsPermission clusterPerms = getClusterDirectoryPermissions(config);
- SliderUtils.copyDirectory(config, snapshotConfPath, generatedConfDirPath,
+ copyDirectory(config, snapshotConfPath, generatedConfDirPath,
clusterPerms);
@@ -2162,15 +2027,15 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// TODO: consider supporting apps that don't have an image path
Path imagePath =
- SliderUtils.extractImagePath(sliderFileSystem, internalOptions);
+ extractImagePath(sliderFileSystem, internalOptions);
if (sliderFileSystem.maybeAddImagePath(localResources, imagePath)) {
log.debug("Registered image path {}", imagePath);
}
// build the environment
amLauncher.putEnv(
- SliderUtils.buildEnvMap(sliderAMResourceComponent));
- ClasspathConstructor classpath = SliderUtils.buildClasspath(relativeConfDir,
+ buildEnvMap(sliderAMResourceComponent));
+ ClasspathConstructor classpath = buildClasspath(relativeConfDir,
libdir,
getConfig(),
sliderFileSystem,
@@ -2182,13 +2047,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
amLauncher.setEnv("LANGUAGE", "en_US.UTF-8");
amLauncher.putEnv(getAmLaunchEnv(config));
- for (Map.Entry<String, String> envs : SliderUtils.getSystemEnv().entrySet()) {
+ for (Map.Entry<String, String> envs : getSystemEnv().entrySet()) {
log.debug("System env {}={}", envs.getKey(), envs.getValue());
}
if (log.isDebugEnabled()) {
log.debug("AM classpath={}", classpath);
log.debug("Environment Map:\n{}",
- SliderUtils.stringifyMap(amLauncher.getEnv()));
+ stringifyMap(amLauncher.getEnv()));
log.debug("Files in lib path\n{}", sliderFileSystem.listFSDir(libPath));
}
@@ -2196,14 +2061,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
InetSocketAddress rmSchedulerAddress;
try {
- rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(config);
+ rmSchedulerAddress = getRmSchedulerAddress(config);
} catch (IllegalArgumentException e) {
throw new BadConfigException("%s Address invalid: %s",
- YarnConfiguration.RM_SCHEDULER_ADDRESS,
- config.get(
- YarnConfiguration.RM_SCHEDULER_ADDRESS)
- );
-
+ YarnConfiguration.RM_SCHEDULER_ADDRESS,
+ config.get(YarnConfiguration.RM_SCHEDULER_ADDRESS));
}
String rmAddr = NetUtils.getHostPortString(rmSchedulerAddress);
@@ -2244,18 +2106,14 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
/**
* pass the registry binding
*/
- addConfOptionToCLI(commandLine, config,
- RegistryConstants.KEY_REGISTRY_ZK_ROOT,
+ commandLine.addConfOptionToCLI(config, RegistryConstants.KEY_REGISTRY_ZK_ROOT,
RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
- addMandatoryConfOptionToCLI(commandLine, config,
- RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
+ commandLine.addMandatoryConfOption(config, RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
if (clusterSecure) {
// if the cluster is secure, make sure that
// the relevant security settings go over
- addConfOptionToCLI(commandLine,
- config,
- DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
+ commandLine.addConfOption(config, DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
}
// write out the path output
commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM);
@@ -2281,8 +2139,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// Set the queue to which this application is to be submitted in the RM
// Queue for App master
String amQueue = config.get(KEY_YARN_QUEUE, DEFAULT_YARN_QUEUE);
- String suppliedQueue = internalOperations.getGlobalOptions().get(InternalKeys.INTERNAL_QUEUE);
- if(!SliderUtils.isUnset(suppliedQueue)) {
+ String suppliedQueue = internalOperations.getGlobalOptions().get(INTERNAL_QUEUE);
+ if(!isUnset(suppliedQueue)) {
amQueue = suppliedQueue;
log.info("Using queue {} for the application instance.", amQueue);
}
@@ -2297,8 +2155,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
protected Map<String, String> getAmLaunchEnv(Configuration config) {
- String sliderAmLaunchEnv = config.get(SliderXmlConfKeys.KEY_AM_LAUNCH_ENV);
- log.debug("{} = {}", SliderXmlConfKeys.KEY_AM_LAUNCH_ENV, sliderAmLaunchEnv);
+ String sliderAmLaunchEnv = config.get(KEY_AM_LAUNCH_ENV);
+ log.debug("{} = {}", KEY_AM_LAUNCH_ENV, sliderAmLaunchEnv);
// Multiple env variables can be specified with a comma (,) separator
String[] envs = StringUtils.isEmpty(sliderAmLaunchEnv) ? null
: sliderAmLaunchEnv.split(",");
@@ -2343,7 +2201,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
String systemKey = placeholderKey
.substring(2, placeholderKey.length() - 1).toUpperCase(Locale.ENGLISH)
.replaceAll("\\.", "_");
- String placeholderValue = SliderUtils.getSystemEnv(systemKey);
+ String placeholderValue = getSystemEnv(systemKey);
log.debug("Placeholder {}={}", placeholderKey, placeholderValue);
placeholderKeyValueMap.put(placeholderKey, placeholderValue);
}
@@ -2353,10 +2211,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private void propagatePythonExecutable(Configuration config,
AggregateConf instanceDefinition) {
String pythonExec = config.get(
- SliderXmlConfKeys.PYTHON_EXECUTABLE_PATH);
+ PYTHON_EXECUTABLE_PATH);
if (pythonExec != null) {
instanceDefinition.getAppConfOperations().getGlobalOptions().putIfUnset(
- SliderXmlConfKeys.PYTHON_EXECUTABLE_PATH,
+ PYTHON_EXECUTABLE_PATH,
pythonExec);
}
}
@@ -2386,7 +2244,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
new Duration(acceptWaitMillis));
// may have failed, so check that
- if (SliderUtils.hasAppFinished(report)) {
+ if (hasAppFinished(report)) {
exitCode = buildExitCode(report);
} else {
// exit unless there is a wait
@@ -2421,68 +2279,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
AggregateConf clusterSpec) {
String dfsPrincipal = config.get(DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
if (dfsPrincipal != null) {
- String siteDfsPrincipal = OptionKeys.SITE_XML_PREFIX +
- DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
+ String siteDfsPrincipal = SITE_XML_PREFIX + DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
clusterSpec.getAppConfOperations().getGlobalOptions().putIfUnset(
siteDfsPrincipal,
dfsPrincipal);
}
}
-
- private boolean addConfOptionToCLI(CommandLineBuilder cmdLine,
- Configuration conf,
- String key) {
- String val = conf.get(key);
- return defineIfSet(cmdLine, key, val);
- }
-
- private String addConfOptionToCLI(CommandLineBuilder cmdLine,
- Configuration conf,
- String key,
- String defVal) {
- String val = conf.get(key, defVal);
- define(cmdLine, key, val);
- return val;
- }
-
- /**
- * Add a <code>-D key=val</code> command to the CLI
- * @param cmdLine command line
- * @param key key
- * @param val value
- */
- private void define(CommandLineBuilder cmdLine, String key, String val) {
- Preconditions.checkArgument(key != null, "null key");
- Preconditions.checkArgument(val != null, "null value");
- cmdLine.add(Arguments.ARG_DEFINE, key + "=" + val);
- }
-
- /**
- * Add a <code>-D key=val</code> command to the CLI if <code>val</code>
- * is not null
- * @param cmdLine command line
- * @param key key
- * @param val value
- */
- private boolean defineIfSet(CommandLineBuilder cmdLine, String key, String val) {
- Preconditions.checkArgument(key != null, "null key");
- if (val != null) {
- define(cmdLine, key, val);
- return true;
- } else {
- return false;
- }
- }
-
- private void addMandatoryConfOptionToCLI(CommandLineBuilder cmdLine,
- Configuration conf,
- String key) throws BadConfigException {
- if (!addConfOptionToCLI(cmdLine, conf, key)) {
- throw new BadConfigException("Missing configuration option: " + key);
- }
- }
-
/**
* Create a path that must exist in the cluster fs
* @param uri uri to create
@@ -2781,9 +2584,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// and those the RM knows about
List<ApplicationReport> instances = listSliderInstances(null);
- SliderUtils.sortApplicationsByMostRecent(instances);
+ sortApplicationsByMostRecent(instances);
Map<String, ApplicationReport> reportMap =
- SliderUtils.buildApplicationReportMap(instances, min, max);
+ buildApplicationReportMap(instances, min, max);
log.debug("Persisted {} deployed {} filtered[{}-{}] & de-duped to {}",
persistentInstances.size(),
instances.size(),
@@ -2816,7 +2619,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// a report
listedInstances.add(name);
// containers will be non-null when only one instance is requested
- String details = SliderUtils.instanceDetailsToString(name, report,
+ String details = instanceDetailsToString(name, report,
containers, version, components, verbose);
print(details);
}
@@ -2899,7 +2702,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
@VisibleForTesting
public int actionFlex(String name, ActionFlexArgs args) throws YarnException, IOException {
verifyBindingsDefined();
- SliderUtils.validateClusterName(name);
+ validateClusterName(name);
log.debug("actionFlex({})", name);
Map<String, Integer> roleInstances = new HashMap<>();
Map<String, String> roleMap = args.getComponentMap();
@@ -2926,7 +2729,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
public int actionExists(String name, ActionExistsArgs args) throws YarnException, IOException {
verifyBindingsDefined();
- SliderUtils.validateClusterName(name);
+ validateClusterName(name);
boolean checkLive = args.live;
log.debug("actionExists({}, {}, {})", name, checkLive, args.state);
@@ -2936,7 +2739,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
throw unknownClusterException(name);
}
String state = args.state;
- if (!checkLive && SliderUtils.isUnset(state)) {
+ if (!checkLive && isUnset(state)) {
log.info("Application {} exists", name);
return EXIT_SUCCESS;
}
@@ -2970,8 +2773,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
}
- SliderUtils.OnDemandReportStringifier report =
- new SliderUtils.OnDemandReportStringifier(instance);
+ OnDemandReportStringifier report =
+ new OnDemandReportStringifier(instance);
if (!inDesiredState) {
//cluster in the list of apps but not running
log.info("Application {} found but is in wrong state {}", name,
@@ -2991,7 +2794,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
public int actionKillContainer(String name,
ActionKillContainerArgs args) throws YarnException, IOException {
String id = args.id;
- if (SliderUtils.isUnset(id)) {
+ if (isUnset(id)) {
throw new BadCommandArgumentsException("Missing container id");
}
log.info("killingContainer {}:{}", name, id);
@@ -3087,7 +2890,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
YarnException,
IOException {
verifyBindingsDefined();
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
String outfile = statusArgs.getOutput();
ClusterDescription status = getClusterDescription(clustername);
String text = status.toJsonString();
@@ -3109,7 +2912,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
public int actionFreeze(String clustername,
ActionFreezeArgs freezeArgs) throws YarnException, IOException {
verifyBindingsDefined();
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
int waittime = freezeArgs.getWaittime();
String text = freezeArgs.message;
boolean forcekill = freezeArgs.force;
@@ -3128,7 +2931,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
log.debug("App to stop was found: {}:\n{}", clustername,
- new SliderUtils.OnDemandReportStringifier(app));
+ new OnDemandReportStringifier(app));
if (app.getYarnApplicationState().ordinal() >=
YarnApplicationState.FINISHED.ordinal()) {
log.info("Cluster {} is in a terminated state {}", clustername,
@@ -3195,7 +2998,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
@Override
public int actionThaw(String clustername, ActionThawArgs thaw) throws YarnException, IOException {
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
verifyBindingsDefined();
// see if it is actually running and bail out;
verifyNoLiveClusters(clustername, "Start");
@@ -3215,7 +3018,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
public int flex(String clustername, Map<String, Integer> roleInstances)
throws YarnException, IOException {
verifyBindingsDefined();
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved(
clustername,
@@ -3226,7 +3029,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
for (Map.Entry<String, Integer> entry : roleInstances.entrySet()) {
String role = entry.getKey();
int count = entry.getValue();
- resources.getOrAddComponent(role).put(ResourceKeys.COMPONENT_INSTANCES,
+ resources.getOrAddComponent(role).put(COMPONENT_INSTANCES,
Integer.toString(count));
log.debug("Flexed cluster specification ( {} -> {}) : \n{}",
@@ -3237,7 +3040,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
SliderAMClientProvider sliderAM = new SliderAMClientProvider(getConfig());
AbstractClientProvider provider = createClientProvider(
instanceDefinition.getInternalOperations().getGlobalOptions().getMandatoryOption(
- InternalKeys.INTERNAL_PROVIDER_NAME));
+ INTERNAL_PROVIDER_NAME));
// slider provider to validate what there is
validateInstanceDefinition(sliderAM, instanceDefinition, sliderFileSystem);
validateInstanceDefinition(provider, instanceDefinition, sliderFileSystem);
@@ -3548,7 +3351,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
} else {
String filename = RegistryPathUtils.lastPathEntry(name) + ".json";
File jsonFile = new File(destDir, filename);
- SliderUtils.write(jsonFile,
+ write(jsonFile,
serviceRecordMarshal.toBytes(instance),
true);
}
@@ -3561,7 +3364,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
outFile = new File(args.destdir, RegistryPathUtils.lastPathEntry(path));
}
if (outFile != null) {
- SliderUtils.write(outFile, serviceRecordMarshal.toBytes(instance), true);
+ write(outFile, serviceRecordMarshal.toBytes(instance), true);
} else {
println(serviceRecordMarshal.toJson(instance));
}
@@ -3592,12 +3395,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
} else if (registryArgs.listExports) {
// list the exports
actionRegistryListExports(registryArgs);
- } else if (SliderUtils.isSet(registryArgs.getConf)) {
+ } else if (isSet(registryArgs.getConf)) {
// get a configuration
PublishedConfiguration publishedConfiguration =
actionRegistryGetConfig(registryArgs);
outputConfig(publishedConfiguration, registryArgs);
- } else if (SliderUtils.isSet(registryArgs.getExport)) {
+ } else if (isSet(registryArgs.getExport)) {
// get a export group
PublishedExports publishedExports =
actionRegistryGetExport(registryArgs);
@@ -3609,11 +3412,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
// JDK7
} catch (FileNotFoundException e) {
- log.info("{}", e);
+ log.info("{}", e.toString());
log.debug("{}", e, e);
return EXIT_NOT_FOUND;
} catch (PathNotFoundException e) {
- log.info("{}", e);
+ log.info("{}", e.toString());
log.debug("{}", e, e);
return EXIT_NOT_FOUND;
}
@@ -3637,10 +3440,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
RegistryOperations operations = getRegistryOperations();
Collection<ServiceRecord> serviceRecords;
if (StringUtils.isEmpty(name)) {
- String path =
- serviceclassPath(
- currentUser(),
- serviceType);
+ String path = serviceclassPath(currentUser(), serviceType);
try {
Map<String, ServiceRecord> recordMap =
@@ -3698,18 +3498,15 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// application name after --application option and member variable
// cluster name has to be put behind action
String clusterName = diagnosticArgs.name;
- if(SliderUtils.isUnset(clusterName)){
- throw new BadCommandArgumentsException("application name must be provided with --name option");
- }
-
+ requireArgumentSet(Arguments.ARG_NAME, clusterName);
+
try {
- SliderUtils.validateClientConfigFile();
+ validateClientConfigFile();
log.info("Slider-client.xml is accessible");
} catch (IOException e) {
// we are catching exceptions here because those are indication of
// validation result, and we need to print them here
- log.error(
- "validation of slider-client.xml fails because: " + e.toString(), e);
+ log.error("validation of slider-client.xml fails because: " + e, e);
return;
}
SliderClusterOperations clusterOperations = createClusterOperations(clusterName);
@@ -3722,7 +3519,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
AggregateConf instanceDefinition = clusterOperations
.getInstanceDefinition();
String imagePath = instanceDefinition.getInternalOperations().get(
- InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ INTERNAL_APPLICATION_IMAGE_PATH);
// if null, that means slider uploaded the agent tarball for the user
// and we need to use where slider has put
if (imagePath == null) {
@@ -3733,26 +3530,27 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
+ "/agent");
imagePath = subPath.toString();
}
+ String pathStr = imagePath + "/" + AGENT_TAR;
try {
- SliderUtils.validateHDFSFile(sliderFileSystem, imagePath + "/" + AGENT_TAR);
+ validateHDFSFile(sliderFileSystem, pathStr);
log.info("Slider agent package is properly installed");
} catch (FileNotFoundException e) {
- log.error("can not find agent package: " + e.toString());
+ log.error("can not find agent package: {}", pathStr);
+ log.debug("can not find agent package: {}", pathStr, e);
return;
} catch (IOException e) {
- log.error("can not open agent package: " + e.toString());
+ log.error("can not open agent package: {}", pathStr, e);
return;
}
- String pkgTarballPath = SliderUtils
- .getApplicationDefinitionPath(instanceDefinition
+ String pkgTarballPath = getApplicationDefinitionPath(instanceDefinition
.getAppConfOperations());
try {
- SliderUtils.validateHDFSFile(sliderFileSystem, pkgTarballPath);
+ validateHDFSFile(sliderFileSystem, pkgTarballPath);
log.info("Application package is properly installed");
} catch (FileNotFoundException e) {
- log.error("can not find application package: {}", e);
+ log.error("can not find application package: {}", pkgTarballPath, e);
} catch (IOException e) {
- log.error("can not open application package: {} ", e);
+ log.error("can not open application package: {} ", pkgTarballPath, e);
}
}
}
@@ -3769,11 +3567,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private void actionDiagnosticCredentials() throws BadConfigException,
IOException {
- if (SliderUtils.isHadoopClusterSecure(SliderUtils
- .loadSliderClientXML())) {
+ if (isHadoopClusterSecure(loadSliderClientXML())) {
String credentialCacheFileDescription = null;
try {
- credentialCacheFileDescription = SliderUtils.checkCredentialCacheFile();
+ credentialCacheFileDescription = checkCredentialCacheFile();
} catch (BadConfigException e) {
log.error("The credential config is not valid: " + e.toString());
throw e;
@@ -3835,25 +3632,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// application name after --application option and member variable
// cluster name has to be put behind action
String clusterName = diagnosticArgs.name;
- if(SliderUtils.isUnset(clusterName)){
+ if(isUnset(clusterName)){
throw new BadCommandArgumentsException("application name must be provided with --name option");
}
- SliderClusterOperations clusterOperations;
- AggregateConf instanceDefinition = null;
- try {
- clusterOperations = createClusterOperations(clusterName);
- instanceDefinition = clusterOperations.getInstanceDefinition();
- } catch (YarnException e) {
- log.error("Exception happened when retrieving instance definition from YARN: "
- + e.toString());
- throw e;
- } catch (IOException e) {
- log.error("Network problem happened when retrieving instance definition from YARN: "
- + e.toString());
- throw e;
- }
+ AggregateConf instanceDefinition = fetchInstanceDefinition(clusterName);
String imagePath = instanceDefinition.getInternalOperations().get(
- InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ INTERNAL_APPLICATION_IMAGE_PATH);
// if null, it will be uploaded by Slider and thus at slider's path
if (imagePath == null) {
ApplicationReport appReport = findInstance(clusterName);
@@ -3865,58 +3649,54 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
log.info("The path of slider agent tarball on HDFS is: " + imagePath);
}
- private void actionDiagnosticApplication(ActionDiagnosticArgs diagnosticArgs)
+ private AggregateConf fetchInstanceDefinition(String clusterName)
throws YarnException, IOException {
- // not using member variable clustername because we want to place
- // application name after --application option and member variable
- // cluster name has to be put behind action
- String clusterName = diagnosticArgs.name;
- if(SliderUtils.isUnset(clusterName)){
- throw new BadCommandArgumentsException("application name must be provided with --name option");
- }
SliderClusterOperations clusterOperations;
AggregateConf instanceDefinition = null;
try {
clusterOperations = createClusterOperations(clusterName);
instanceDefinition = clusterOperations.getInstanceDefinition();
- } catch (YarnException e) {
- log.error("Exception happened when retrieving instance definition from YARN: "
- + e.toString());
- throw e;
- } catch (IOException e) {
- log.error("Network problem happened when retrieving instance definition from YARN: "
+ } catch (YarnException | IOException e) {
+ log.error("Failed to retrieve instance definition from YARN: "
+ e.toString());
throw e;
}
+ return instanceDefinition;
+ }
+
+ private void actionDiagnosticApplication(ActionDiagnosticArgs diagnosticArgs)
+ throws YarnException, IOException {
+ // not using member variable clustername because we want to place
+ // application name after --application option and member variable
+ // cluster name has to be put behind action
+ String clusterName = diagnosticArgs.name;
+ requireArgumentSet(Arguments.ARG_NAME, clusterName);
+ AggregateConf instanceDefinition = fetchInstanceDefinition(clusterName);
String clusterDir = instanceDefinition.getAppConfOperations()
.getGlobalOptions().get(AgentKeys.APP_ROOT);
- String pkgTarball = SliderUtils
- .getApplicationDefinitionPath(instanceDefinition.getAppConfOperations());
+ String pkgTarball = getApplicationDefinitionPath(instanceDefinition.getAppConfOperations());
String runAsUser = instanceDefinition.getAppConfOperations()
.getGlobalOptions().get(AgentKeys.RUNAS_USER);
- log.info("The location of the cluster instance directory in HDFS is: "
- + clusterDir);
- log.info("The name of the application package tarball on HDFS is: "
- + pkgTarball);
- log.info("The runas user of the application in the cluster is: "
- + runAsUser);
+ log.info("The location of the cluster instance directory in HDFS is: {}", clusterDir);
+ log.info("The name of the application package tarball on HDFS is: {}",pkgTarball);
+ log.info("The runas user of the application in the cluster is: {}",runAsUser);
if (diagnosticArgs.verbose) {
- log.info("App config of the application: "
- + instanceDefinition.getAppConf().toJson());
- log.info("Resource config of the application: "
- + instanceDefinition.getResources().toJson());
+ log.info("App config of the application:\n{}",
+ instanceDefinition.getAppConf().toJson());
+ log.info("Resource config of the application:\n{}",
+ instanceDefinition.getResources().toJson());
}
}
private void actionDiagnosticClient(ActionDiagnosticArgs diagnosticArgs)
throws SliderException, IOException {
try {
- String currentCommandPath = SliderUtils.getCurrentCommandPath();
+ String currentCommandPath = getCurrentCommandPath();
SliderVersionInfo.loadAndPrintVersionInfo(log);
- String clientConfigPath = SliderUtils.getClientConfigPath();
- String jdkInfo = SliderUtils.getJDKInfo();
+ String clientConfigPath = getClientConfigPath();
+ String jdkInfo = getJDKInfo();
println("The slider command path: %s", currentCommandPath);
println("The slider-client.xml used by current running command path: %s",
clientConfigPath);
@@ -3924,7 +3704,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// security info
Configuration config = getConfig();
- if (SliderUtils.isHadoopClusterSecure(config)) {
+ if (isHadoopClusterSecure(config)) {
println("Hadoop Cluster is secure");
println("Login user is %s", UserGroupInformation.getLoginUser());
println("Current user is %s", UserGroupInformation.getCurrentUser());
@@ -3936,7 +3716,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// verbose?
if (diagnosticArgs.verbose) {
// do the environment
- Map<String, String> env = SliderUtils.getSystemEnv();
+ Map<String, String> env = getSystemEnv();
Set<String> envList = ConfigHelper.sortedConfigKeys(env.entrySet());
StringBuilder builder = new StringBuilder("Environment variables:\n");
for (String key : envList) {
@@ -3947,7 +3727,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// Java properties
builder = new StringBuilder("JVM Properties\n");
Map<String, String> props =
- SliderUtils.sortedMap(SliderUtils.toMap(System.getProperties()));
+ sortedMap(toMap(System.getProperties()));
for (Entry<String, String> entry : props.entrySet()) {
builder.append(entry.getKey()).append("=")
.append(entry.getValue()).append("\n");
@@ -3956,12 +3736,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
println(builder.toString());
// then the config
- println("Slider client configuration:\n"
- + ConfigHelper.dumpConfigToString(config));
-
+ println("Slider client configuration:\n" + ConfigHelper.dumpConfigToString(config));
}
- SliderUtils.validateSliderClientEnvironment(log);
+ validateSliderClientEnvironment(log);
} catch (SliderException | IOException e) {
log.error(e.toString());
throw e;
@@ -4122,8 +3900,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
RegistryRetriever retriever = new RegistryRetriever(getConfig(), instance);
boolean external = !registryArgs.internal;
- PublishedExportsSet exports =
- retriever.getExports(external);
+ PublishedExportsSet exports = retriever.getExports(external);
PublishedExports published = retriever.retrieveExports(exports,
registryArgs.getExport,
@@ -4395,7 +4172,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_FALSE;
}
- String version = SliderUtils.getSliderVersion();
+ String version = getSliderVersion();
Path dependencyLibTarGzip = sliderFileSystem.getDependencyTarGzip();
// Check if dependency has already been uploaded, in which case log
@@ -4409,23 +4186,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
String libDir = System.getProperty(SliderKeys.PROPERTY_LIB_DIR);
- if (SliderUtils.isSet(libDir)) {
+ if (isSet(libDir)) {
File srcFolder = new File(libDir);
File tempLibTarGzipFile = File.createTempFile(
SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME + "_",
SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT);
// copy all jars except slider-core-<version>.jar
- FilenameFilter jarFilter = new FilenameFilter() {
- public boolean accept(File dir, String name) {
- String lowercaseName = name.toLowerCase();
- if (lowercaseName.endsWith(".jar")) {
- return true;
- } else {
- return false;
- }
- }
- };
- SliderUtils.tarGzipFolder(srcFolder, tempLibTarGzipFile, jarFilter);
+ tarGzipFolder(srcFolder, tempLibTarGzipFile, createJarFilter());
log.info("Uploading dependency for AM (version {}) from {} to {}",
version, tempLibTarGzipFile.toURI(), dependencyLibTarGzip.toUri());
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index d34dd2a..a1ad690 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -159,7 +159,7 @@ public class CoreFileSystem {
Preconditions.checkNotNull(packageName);
Path path = getBaseApplicationPath();
path = new Path(path, SliderKeys.PACKAGE_DIRECTORY + "/" + packageName);
- if (StringUtils.isNotEmpty(packageVersion)) {
+ if (SliderUtils.isSet(packageVersion)) {
path = new Path(path, packageVersion);
}
return path;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index f380ec6..1f97982 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -1034,7 +1034,7 @@ public final class SliderUtils {
* something other than 0.0.0.0
*/
public static boolean isAddressDefined(InetSocketAddress address) {
- return !(address.getHostName().equals("0.0.0.0"));
+ return !(address.getHostString().equals("0.0.0.0"));
}
public static void setRmAddress(Configuration conf, String rmAddr) {
@@ -1352,16 +1352,7 @@ public final class SliderUtils {
log.info("Loading all dependencies from {}", srcPath);
if (SliderUtils.isSet(srcPath)) {
File srcFolder = new File(srcPath);
- FilenameFilter jarFilter = new FilenameFilter() {
- public boolean accept(File dir, String name) {
- String lowercaseName = name.toLowerCase();
- if (lowercaseName.endsWith(".jar")) {
- return true;
- } else {
- return false;
- }
- }
- };
+ FilenameFilter jarFilter = createJarFilter();
File[] listOfJars = srcFolder.listFiles(jarFilter);
for (File jarFile : listOfJars) {
LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName());
@@ -1371,6 +1362,18 @@ public final class SliderUtils {
}
/**
+ * Accept all filenames ending with {@code .jar}
+ * @return a filename filter
+ */
+ public static FilenameFilter createJarFilter() {
+ return new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.toLowerCase(Locale.ENGLISH).endsWith(".jar");
+ }
+ };
+ }
+
+ /**
* Submit the AM tar.gz containing all dependencies and map it
* @param providerResources provider map to build up
* @param sliderFileSystem remote fs
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
index b4ca791..06dbfea 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
@@ -220,8 +220,7 @@ public class AppMasterLauncher extends AbstractLauncher {
}
Token<? extends TokenIdentifier>[] tokens = null;
- boolean tokensProvided =
- this.getConf().get(MAPREDUCE_JOB_CREDENTIALS_BINARY) != null;
+ boolean tokensProvided = getConf().get(MAPREDUCE_JOB_CREDENTIALS_BINARY) != null;
if (!tokensProvided) {
// For now, only getting tokens for the default file-system.
FileSystem fs = coreFileSystem.getFileSystem();
@@ -232,7 +231,7 @@ public class AppMasterLauncher extends AbstractLauncher {
if (tokens != null && tokens.length > 0) {
AbstractDelegationTokenIdentifier id =
(AbstractDelegationTokenIdentifier)tokens[0].decodeIdentifier();
- Date d = new Date(id.getIssueDate() + 24*60*60*1000);
+ Date d = new Date(id.getIssueDate() + 24 * 60 * 60 * 1000);
log.info("HDFS delegation tokens for AM launch context require renewal by {}",
DateFormat.getDateTimeInstance().format(d));
} else {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
index 3ba0181..6eb4058 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
@@ -38,7 +38,6 @@ import java.util.List;
public class ClasspathConstructor {
public static final String CLASS_PATH_SEPARATOR = ApplicationConstants.CLASS_PATH_SEPARATOR;
-// public static final String CLASS_PATH_SEPARATOR = File.pathSeparator;
private final List<String> pathElements = new ArrayList<>();
public ClasspathConstructor() {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java b/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
index dbaa981..57b8965 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
@@ -19,8 +19,11 @@
package org.apache.slider.core.launch;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.slider.common.params.Arguments;
import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
import java.util.ArrayList;
import java.util.List;
@@ -101,4 +104,56 @@ public class CommandLineBuilder {
return argumentList;
}
+ public boolean addConfOption(Configuration conf, String key) {
+ String val = conf.get(key);
+ return defineIfSet(key, val);
+ }
+
+ public String addConfOptionToCLI(Configuration conf,
+ String key,
+ String defVal) {
+ String val = conf.get(key, defVal);
+ define(key, val);
+ return val;
+ }
+
+ /**
+ * Add a <code>-D key=val</code> command to the CLI
+ * @param key key
+ * @param val value
+ */
+ public void define(String key, String val) {
+ Preconditions.checkArgument(key != null, "null key");
+ Preconditions.checkArgument(val != null, "null value");
+ add(Arguments.ARG_DEFINE, key + "=" + val);
+ }
+
+ /**
+ * Add a <code>-D key=val</code> command to the CLI if <code>val</code>
+ * is not null
+ * @param key key
+ * @param val value
+ */
+ public boolean defineIfSet(String key, String val) {
+ Preconditions.checkArgument(key != null, "null key");
+ if (val != null) {
+ define(key, val);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Add a mandatory config option
+ * @param conf configuration
+ * @param key key
+ * @throws BadConfigException if the key is missing
+ */
+ public void addMandatoryConfOption(Configuration conf,
+ String key) throws BadConfigException {
+ if (!addConfOption(conf, key)) {
+ throw new BadConfigException("Missing configuration option: " + key);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
index 7f3931b..1622309 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
@@ -98,11 +98,23 @@ public abstract class AbstractSliderLaunchedService extends
*/
protected static void requireArgumentSet(String argname, String value)
throws BadCommandArgumentsException {
- if (isUnset(value)) {
- throw new BadCommandArgumentsException(
- "Required argument " + argname + " missing");
- }
+ require(isSet(value), "Required argument %s missing", argname );
}
+ /**
+ * Require a condition to hold; throw {@link BadCommandArgumentsException} if not.
+ * The exception text is the formatted message.
+ * @param condition condition
+ * @param message string to format
+ * @param args list of arguments to format.
+ * @throws BadCommandArgumentsException
+ */
+ protected static void require(boolean condition, String message,
+ Object... args)
+ throws BadCommandArgumentsException {
+ if (!condition) {
+ throw new BadCommandArgumentsException(message, args);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy
index 8245dae..7e3f55a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy
@@ -41,8 +41,7 @@ import org.junit.Test
class TestActionPackage extends AgentMiniClusterTestBase {
- public static final String E_INVALID_APP_TYPE =
- "A valid application type name is required (e.g. HBASE)"
+ public static final String E_NO_NAME = "Required argument --name missing"
String s = File.separator
File packageFile = new File("src${s}test${s}resources${s}log4j.properties")
@@ -70,7 +69,7 @@ class TestActionPackage extends AgentMiniClusterTestBase {
} catch (BadCommandArgumentsException e) {
assertExceptionDetails(e,
LauncherExitCodes.EXIT_COMMAND_ARGUMENT_ERROR,
- E_INVALID_APP_TYPE)
+ E_NO_NAME)
}
}
@@ -285,7 +284,7 @@ class TestActionPackage extends AgentMiniClusterTestBase {
} catch (BadCommandArgumentsException e) {
assertExceptionDetails(e,
LauncherExitCodes.EXIT_COMMAND_ARGUMENT_ERROR,
- E_INVALID_APP_TYPE)
+ E_NO_NAME)
}
}
}