You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/11/04 17:42:38 UTC
incubator-slider git commit: SLIDER-82 preparing ground for
anti-affinity. * app state binding moves from multiple args to a single
binding struct;
this significantly simplifies test setup * references in roleHistory to
available nodes are replace with "
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-82-pass-3 [created] 4e9a95b92
SLIDER-82 preparing ground for anti-affinity.
* app state binding moves from multiple args to a single binding struct; this significantly simplifies test setup
* references in roleHistory to available nodes are replace with "recent", as that is what they currently are
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4e9a95b9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4e9a95b9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4e9a95b9
Branch: refs/heads/feature/SLIDER-82-pass-3
Commit: 4e9a95b926b0b1bae2d210f63fcb592043be6091
Parents: c8fb17e
Author: Steve Loughran <st...@apache.org>
Authored: Wed Nov 4 16:35:03 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Nov 4 16:35:03 2015 +0000
----------------------------------------------------------------------
.../slider/client/SliderYarnClientImpl.java | 9 +-
.../slider/providers/PlacementPolicy.java | 7 +-
.../server/appmaster/SliderAppMaster.java | 93 ++++++++++----------
.../slider/server/appmaster/state/AppState.java | 81 +++++++----------
.../appmaster/state/AppStateBindingInfo.java | 59 +++++++++++++
.../server/appmaster/state/RoleHistory.java | 79 +++++++++--------
.../TestMockAppStateDynamicHistory.groovy | 28 +-----
.../TestMockAppStateDynamicRoles.groovy | 33 +++----
.../TestMockAppStateFlexDynamicRoles.groovy | 44 ++++-----
.../TestMockAppStateRebuildOnAMRestart.groovy | 32 +++----
.../TestMockContainerResourceAllocations.groovy | 5 +-
.../TestRoleHistoryContainerEvents.groovy | 2 +-
...stRoleHistoryFindNodesForNewInstances.groovy | 2 +-
.../model/history/TestRoleHistoryRW.groovy | 4 +-
.../TestRoleHistoryRequestTracking.groovy | 4 +-
.../model/mock/BaseMockAppStateTest.groovy | 41 +++++----
.../appmaster/model/mock/MockAppState.groovy | 10 ++-
.../web/rest/agent/TestAMAgentWebServices.java | 47 ++++------
.../management/TestAMManagementWebServices.java | 71 +++++----------
19 files changed, 302 insertions(+), 349 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e9a95b9/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index 3b7a65c..42759fd 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -55,8 +55,7 @@ import java.util.Set;
* from the slider entry point service
*/
public class SliderYarnClientImpl extends YarnClientImpl {
- protected static final Logger
- log = LoggerFactory.getLogger(SliderYarnClientImpl.class);
+ protected static final Logger log = LoggerFactory.getLogger(SliderYarnClientImpl.class);
/**
* Keyword to use in the {@link #emergencyForceKill(String)}
@@ -96,10 +95,10 @@ public class SliderYarnClientImpl extends YarnClientImpl {
public List<ApplicationReport> listDeployedInstances(String user)
throws YarnException, IOException {
Preconditions.checkArgument(user != null, "Null User");
- Set<String> types = new HashSet<String>(1);
+ Set<String> types = new HashSet<>(1);
types.add(SliderKeys.APP_TYPE);
List<ApplicationReport> allApps = getApplications(types);
- List<ApplicationReport> results = new ArrayList<ApplicationReport>();
+ List<ApplicationReport> results = new ArrayList<>();
for (ApplicationReport report : allApps) {
if (StringUtils.isEmpty(user) || user.equals(report.getUser())) {
results.add(report);
@@ -330,8 +329,6 @@ public class SliderYarnClientImpl extends YarnClientImpl {
Preconditions.checkArgument(StringUtils.isNotEmpty(appname),
"Null/empty application name");
Preconditions.checkArgument(desiredState != null, "Null desiredState");
- ApplicationReport found = null;
- ApplicationReport foundAndLive = null;
log.debug("Searching {} records for instance name {} in state '{}'",
instances.size(), appname, desiredState);
for (ApplicationReport app : instances) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e9a95b9/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java b/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
index 4e85a93..e0913a5 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
@@ -27,7 +27,12 @@ public class PlacementPolicy {
/**
* Default value: history used, anti-affinity hinted at on rebuild/flex up
*/
- public static final int DEFAULT = 0;
+ public static final int NONE = 0;
+
+ /**
+ * Default value: history used, anti-affinity hinted at on rebuild/flex up
+ */
+ public static final int DEFAULT = NONE;
/**
* Strict placement: when asking for an instance for which there is
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e9a95b9/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index df91d7f..b552290 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -59,13 +59,16 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
@@ -84,6 +87,7 @@ import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.RoleKeys;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.api.proto.SliderClusterAPI;
+import org.apache.slider.client.SliderYarnClientImpl;
import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.params.AbstractActionArgs;
@@ -143,6 +147,7 @@ import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
import org.apache.slider.server.appmaster.rpc.SliderIPCService;
import org.apache.slider.server.appmaster.security.SecurityConfiguration;
import org.apache.slider.server.appmaster.state.AppState;
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
import org.apache.slider.server.appmaster.state.ContainerAssignment;
import org.apache.slider.server.appmaster.state.ProviderAppState;
import org.apache.slider.server.appmaster.operations.RMOperationHandler;
@@ -356,16 +361,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
private RegistryOperations registryOperations;
/**
- * Record of the max no. of cores allowed in this cluster
- */
- private int containerMaxCores;
-
- /**
- * limit container memory
- */
- private int containerMaxMemory;
-
- /**
* The stop request received...the exit details are extracted
* from this
*/
@@ -415,6 +410,12 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
*/
private boolean securityEnabled;
private ContentCache contentCache;
+ private SliderYarnClientImpl yarnClient;
+
+ /**
+ * resource limits
+ */
+ private Resource maximumResourceCapability;
/**
* Service Constructor
@@ -437,10 +438,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// Load in the server configuration - if it is actually on the Classpath
URL serverXmlUrl = ConfigHelper.getResourceUrl(SLIDER_SERVER_XML);
if (serverXmlUrl != null) {
-
log.info("Loading {} at {}", SLIDER_SERVER_XML, serverXmlUrl);
- Configuration serverConf =
- ConfigHelper.loadFromResource(SLIDER_SERVER_XML);
+ Configuration serverConf = ConfigHelper.loadFromResource(SLIDER_SERVER_XML);
ConfigHelper.mergeConfigurations(customConf, serverConf,
SLIDER_SERVER_XML, true);
}
@@ -505,7 +504,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
addService(executorService);
addService(actionQueues);
-
+ addService(yarnClient = new SliderYarnClientImpl());
+
//init all child services
super.serviceInit(conf);
}
@@ -630,8 +630,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
Configuration serviceConf = getConfig();
- securityConfiguration = new SecurityConfiguration(
- serviceConf, instanceDefinition, clustername);
+ securityConfiguration = new SecurityConfiguration(serviceConf, instanceDefinition, clustername);
// obtain security state
securityEnabled = securityConfiguration.isSecurityEnabled();
// set the global security flag for the instance definition
@@ -659,8 +658,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
providerService = factory.createServerProvider();
// init the provider BUT DO NOT START IT YET
initAndAddService(providerService);
- providerRMOperationHandler =
- new ProviderNotifyingOperationHandler(providerService);
+ providerRMOperationHandler = new ProviderNotifyingOperationHandler(providerService);
// create a slider AM provider
sliderAMProvider = new SliderAMProviderService();
@@ -682,12 +680,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
ApplicationId appid = appAttemptID.getApplicationId();
log.info("AM for ID {}", appid.getId());
- appInformation.put(StatusKeys.INFO_AM_CONTAINER_ID,
- appMasterContainerID.toString());
- appInformation.put(StatusKeys.INFO_AM_APP_ID,
- appid.toString());
- appInformation.put(StatusKeys.INFO_AM_ATTEMPT_ID,
- appAttemptID.toString());
+ appInformation.put(StatusKeys.INFO_AM_CONTAINER_ID, appMasterContainerID.toString());
+ appInformation.put(StatusKeys.INFO_AM_APP_ID, appid.toString());
+ appInformation.put(StatusKeys.INFO_AM_ATTEMPT_ID, appAttemptID.toString());
Map<String, String> envVars;
List<Container> liveContainers;
@@ -715,7 +710,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
if (securityEnabled) {
// fix up the ACLs if they are not set
- String acls = getConfig().get(KEY_PROTOCOL_ACL);
+ String acls = serviceConf.get(KEY_PROTOCOL_ACL);
if (acls == null) {
getConfig().set(KEY_PROTOCOL_ACL, "*");
}
@@ -731,8 +726,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
appMasterHostname = rpcServiceAddress.getHostName();
appMasterRpcPort = rpcServiceAddress.getPort();
appMasterTrackingUrl = null;
- log.info("AM Server is listening at {}:{}", appMasterHostname,
- appMasterRpcPort);
+ log.info("AM Server is listening at {}:{}", appMasterHostname, appMasterRpcPort);
appInformation.put(StatusKeys.INFO_AM_HOSTNAME, appMasterHostname);
appInformation.set(StatusKeys.INFO_AM_RPC_PORT, appMasterRpcPort);
@@ -776,24 +770,27 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// *****************************************************
log.info("Connecting to RM at {},address tracking URL={}",
appMasterRpcPort, appMasterTrackingUrl);
- amRegistrationData = asyncRMClient
- .registerApplicationMaster(appMasterHostname,
+ amRegistrationData = asyncRMClient.registerApplicationMaster(appMasterHostname,
appMasterRpcPort,
appMasterTrackingUrl);
- Resource maxResources =
- amRegistrationData.getMaximumResourceCapability();
- containerMaxMemory = maxResources.getMemory();
- containerMaxCores = maxResources.getVirtualCores();
- appState.setContainerLimits(maxResources.getMemory(),
- maxResources.getVirtualCores());
+ maximumResourceCapability = amRegistrationData.getMaximumResourceCapability();
+
+ int minMemory = serviceConf.getInt(RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ // validate scheduler vcores allocation setting
+ int minCores = serviceConf.getInt(RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+ DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
+ int maxMemory = maximumResourceCapability.getMemory();
+ int maxCores = maximumResourceCapability.getVirtualCores();
+ appState.setContainerLimits(minMemory,maxMemory, minCores, maxCores );
// build the handler for RM request/release operations; this uses
// the max value as part of its lookup
- rmOperationHandler = new AsyncRMOperationHandler(asyncRMClient, maxResources);
+ rmOperationHandler = new AsyncRMOperationHandler(asyncRMClient, maximumResourceCapability);
// set the RM-defined maximum cluster values
- appInformation.put(ResourceKeys.YARN_CORES, Integer.toString(containerMaxCores));
- appInformation.put(ResourceKeys.YARN_MEMORY, Integer.toString(containerMaxMemory));
+ appInformation.put(ResourceKeys.YARN_CORES, Integer.toString(maxCores));
+ appInformation.put(ResourceKeys.YARN_MEMORY, Integer.toString(maxMemory));
processAMCredentials(securityConfiguration);
@@ -837,15 +834,17 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
Path historyDir = new Path(clusterDirPath, HISTORY_DIR_NAME);
//build the instance
- appState.buildInstance(instanceDefinition,
- serviceConf,
- providerConf,
- providerRoles,
- fs.getFileSystem(),
- historyDir,
- liveContainers,
- appInformation,
- providerService.createContainerReleaseSelector());
+ AppStateBindingInfo binding = new AppStateBindingInfo();
+ binding.instanceDefinition = instanceDefinition;
+ binding.serviceConfig = serviceConf;
+ binding.publishedProviderConf = providerConf;
+ binding.roles = providerRoles;
+ binding.fs = fs.getFileSystem();
+ binding.historyPath = historyDir;
+ binding.liveContainers = liveContainers;
+ binding.applicationInfo = appInformation;
+ binding.releaseSelector = providerService.createContainerReleaseSelector();
+ appState.buildInstance(binding);
providerService.rebuildContainerDetails(liveContainers,
instanceDefinition.getName(), appState.getRolePriorityMap());
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e9a95b9/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index eadb1dc..1325148 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -21,7 +21,6 @@ package org.apache.slider.server.appmaster.state;
import com.codahale.metrics.Counter;
import com.codahale.metrics.MetricRegistry;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -265,15 +264,17 @@ public class AppState {
/**
- * Record of the max no. of cores allowed in this cluster
+ * limits of container core numbers in this queue
*/
private int containerMaxCores;
+ private int containerMinCores;
/**
- * limit container memory
+ * limits of container memory in this queue
*/
private int containerMaxMemory;
-
+ private int containerMinMemory;
+
private RoleHistory roleHistory;
private Configuration publishedProviderConf;
private long startTimeThreshold;
@@ -447,27 +448,29 @@ public class AppState {
}
/**
- * Set the container limits -the max that can be asked for,
- * which are used when the "max" values are requested
+ * Set the container limits -the min and max values for
+ * resource requests. All requests must be multiples of the min
+ * values.
+ * @param minMemory min memory MB
* @param maxMemory maximum memory
+ * @param minCores min v core count
* @param maxCores maximum cores
*/
- public void setContainerLimits(int maxMemory, int maxCores) {
+ public void setContainerLimits(int minMemory,int maxMemory, int minCores, int maxCores) {
+ containerMinCores = minCores;
containerMaxCores = maxCores;
+ containerMinMemory = minMemory;
containerMaxMemory = maxMemory;
}
-
public ConfTreeOperations getResourcesSnapshot() {
return resourcesSnapshot;
}
-
public ConfTreeOperations getAppConfSnapshot() {
return appConfSnapshot;
}
-
public ConfTreeOperations getInternalsSnapshot() {
return internalsSnapshot;
}
@@ -488,38 +491,17 @@ public class AppState {
return unresolvedInstanceDefinition;
}
- /**
- * Build up the application state
- * @param instanceDefinition definition of the applicatin instance
- * @param appmasterConfig
- * @param publishedProviderConf any configuration info to be published by a provider
- * @param providerRoles roles offered by a provider
- * @param fs filesystem
- * @param historyDir directory containing history files
- * @param liveContainers list of live containers supplied on an AM restart
- * @param applicationInfo app info to retain for web views
- * @param releaseSelector selector of containers to release
- */
- public synchronized void buildInstance(AggregateConf instanceDefinition,
- Configuration appmasterConfig,
- Configuration publishedProviderConf,
- List<ProviderRole> providerRoles,
- FileSystem fs,
- Path historyDir,
- List<Container> liveContainers,
- Map<String, String> applicationInfo,
- ContainerReleaseSelector releaseSelector)
- throws BadClusterStateException, BadConfigException, IOException {
- Preconditions.checkArgument(instanceDefinition != null);
- Preconditions.checkArgument(releaseSelector != null);
+ public synchronized void buildInstance(AppStateBindingInfo binding)
+ throws BadClusterStateException, BadConfigException, IOException {
+ binding.validate();
log.debug("Building application state");
- this.publishedProviderConf = publishedProviderConf;
- this.applicationInfo = applicationInfo != null ? applicationInfo
- : new HashMap<String, String>();
+ publishedProviderConf = binding.publishedProviderConf;
+ applicationInfo = binding.applicationInfo != null ? binding.applicationInfo
+ : new HashMap<String, String>();
clientProperties = new HashMap<>();
- containerReleaseSelector = releaseSelector;
+ containerReleaseSelector = binding.releaseSelector;
Set<String> confKeys = ConfigHelper.sortedConfigKeys(publishedProviderConf);
@@ -532,15 +514,15 @@ public class AppState {
// set the cluster specification (once its dependency the client properties
// is out the way
- setInitialInstanceDefinition(instanceDefinition);
+ setInitialInstanceDefinition(binding.instanceDefinition);
//build the initial role list
- for (ProviderRole providerRole : providerRoles) {
+ List<ProviderRole> roleList = new ArrayList<>(binding.roles);
+ for (ProviderRole providerRole : roleList) {
buildRole(providerRole);
}
- ConfTreeOperations resources =
- instanceDefinition.getResourceOperations();
+ ConfTreeOperations resources = instanceDefinition.getResourceOperations();
Set<String> roleNames = resources.getComponentNames();
for (String name : roleNames) {
@@ -551,16 +533,14 @@ public class AppState {
ProviderRole dynamicRole =
createDynamicProviderRole(name, resComponent);
buildRole(dynamicRole);
- providerRoles.add(dynamicRole);
+ roleList.add(dynamicRole);
}
}
//then pick up the requirements
buildRoleRequirementsFromResources();
-
//set the livespan
- MapOperations globalResOpts =
- instanceDefinition.getResourceOperations().getGlobalOptions();
+ MapOperations globalResOpts = instanceDefinition.getResourceOperations().getGlobalOptions();
startTimeThreshold = globalResOpts.getOptionInt(
InternalKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE,
@@ -576,16 +556,15 @@ public class AppState {
// set up the role history
- roleHistory = new RoleHistory(providerRoles);
+ roleHistory = new RoleHistory(roleList);
roleHistory.register(metricsAndMonitoring);
- roleHistory.onStart(fs, historyDir);
+ roleHistory.onStart(binding.fs, binding.historyPath);
//rebuild any live containers
- rebuildModelFromRestart(liveContainers);
+ rebuildModelFromRestart(binding.liveContainers);
// any am config options to pick up
- logServerURL = appmasterConfig.get(YarnConfiguration.YARN_LOG_SERVER_URL, "");
-
+ logServerURL = binding.serviceConfig.get(YarnConfiguration.YARN_LOG_SERVER_URL, "");
//mark as live
applicationLive = true;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e9a95b9/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
new file mode 100644
index 0000000..184c8aa
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.providers.ProviderRole;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Binding information for application states; designed to be extensible
+ * so that tests don't have to be massivley reworked when new arguments
+ * are added.
+ */
+public class AppStateBindingInfo {
+ public AggregateConf instanceDefinition;
+ public Configuration serviceConfig = new Configuration();
+ public Configuration publishedProviderConf = new Configuration(false);
+ public List<ProviderRole> roles = new ArrayList<>();
+ public FileSystem fs;
+ public Path historyPath;
+ public List<Container> liveContainers = new ArrayList<>(0);
+ public Map<String, String> applicationInfo = new HashMap<>();
+ public ContainerReleaseSelector releaseSelector = new SimpleReleaseSelector();
+
+ public void validate() throws IllegalArgumentException {
+ Preconditions.checkArgument(instanceDefinition != null, "null instanceDefinition");
+ Preconditions.checkArgument(serviceConfig != null, "null appmasterConfig");
+ Preconditions.checkArgument(publishedProviderConf != null, "null publishedProviderConf");
+ Preconditions.checkArgument(releaseSelector != null, "null releaseSelector");
+ Preconditions.checkArgument(roles != null, "null providerRoles");
+ Preconditions.checkArgument(fs != null, "null fs");
+ Preconditions.checkArgument(historyPath != null, "null historyDir");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e9a95b9/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index df3983a..d9a6b34 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -31,7 +31,6 @@ import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.server.appmaster.management.BoolMetric;
-import org.apache.slider.server.appmaster.management.LongGauge;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.management.Timestamp;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
@@ -52,6 +51,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
/**
* The Role History.
@@ -87,8 +87,8 @@ public class RoleHistory {
private RoleHistoryWriter historyWriter = new RoleHistoryWriter();
/**
- * When were the nodes updated in a {@link #onNodesUpdated(List)} call.
- * If zero: never
+ * When were the nodes updated in a {@link #onNodesUpdated(List)} call?
+ * If zero: never.
*/
private final Timestamp nodesUpdatedTime = new Timestamp(0);
private final BoolMetric nodeUpdateReceived = new BoolMetric(false);
@@ -98,9 +98,10 @@ public class RoleHistory {
/**
* For each role, lists nodes that are available for data-local allocation,
- ordered by more recently released - To accelerate node selection
+ * ordered by more recently released - to accelerate node selection.
+ * That is, they are "recently used nodes"
*/
- private Map<Integer, LinkedList<NodeInstance>> availableNodes;
+ private Map<Integer, LinkedList<NodeInstance>> recentNodes;
/**
* Track the failed nodes. Currently used to make wiser decision of container
@@ -158,8 +159,7 @@ public class RoleHistory {
throws BadConfigException {
int index = providerRole.id;
if (index < 0) {
- throw new BadConfigException("Provider " + providerRole
- + " id is out of range");
+ throw new BadConfigException("Provider " + providerRole + " id is out of range");
}
if (roleStats.get(index) != null) {
throw new BadConfigException(
@@ -206,7 +206,7 @@ public class RoleHistory {
* Clear the lists of available nodes
*/
private synchronized void resetAvailableNodeLists() {
- availableNodes = new HashMap<>(roleSize);
+ recentNodes = new ConcurrentHashMap<>(roleSize);
}
/**
@@ -363,7 +363,7 @@ public class RoleHistory {
public synchronized void insert(Collection<NodeInstance> nodes) {
nodemap.insert(nodes);
}
-
+
/**
* Get current time. overrideable for test subclasses
* @return current time in millis
@@ -435,8 +435,7 @@ public class RoleHistory {
* @param historyDir path in FS for history
* @return true if the history was thawed
*/
- public boolean onStart(FileSystem fs, Path historyDir) throws
- BadConfigException {
+ public boolean onStart(FileSystem fs, Path historyDir) throws BadConfigException {
assert filesystem == null;
filesystem = fs;
historyPath = historyDir;
@@ -483,7 +482,7 @@ public class RoleHistory {
}
//start is then completed
- buildAvailableNodeLists();
+ buildRecentNodeLists();
} else {
//fallback to bootstrap procedure
onBootstrap();
@@ -496,7 +495,7 @@ public class RoleHistory {
* (After the start), rebuild the availability data structures
*/
@VisibleForTesting
- public synchronized void buildAvailableNodeLists() {
+ public synchronized void buildRecentNodeLists() {
resetAvailableNodeLists();
// build the list of available nodes
for (Map.Entry<String, NodeInstance> entry : nodemap.entrySet()) {
@@ -505,13 +504,13 @@ public class RoleHistory {
NodeEntry nodeEntry = ni.get(i);
if (nodeEntry != null && nodeEntry.isAvailable()) {
log.debug("Adding {} for role {}", ni, i);
- getOrCreateNodesForRoleId(i).add(ni);
+ listRecentNodesForRoleId(i).add(ni);
}
}
}
// sort the resulting arrays
for (int i = 0; i < roleSize; i++) {
- sortAvailableNodeList(i);
+ sortRecentNodeList(i);
}
}
@@ -521,30 +520,35 @@ public class RoleHistory {
* @return potentially null list
*/
@VisibleForTesting
- public List<NodeInstance> getNodesForRoleId(int id) {
- return availableNodes.get(id);
+ public List<NodeInstance> getRecentNodesForRoleId(int id) {
+ return recentNodes.get(id);
}
-
+
/**
- * Get the nodes for an ID -may be null
+ * Get a possibly emtpy list of suggested nodes for a role.
* @param id role ID
* @return list
*/
- private LinkedList<NodeInstance> getOrCreateNodesForRoleId(int id) {
- LinkedList<NodeInstance> instances = availableNodes.get(id);
+ private LinkedList<NodeInstance> listRecentNodesForRoleId(int id) {
+ LinkedList<NodeInstance> instances = recentNodes.get(id);
if (instances == null) {
- instances = new LinkedList<>();
- availableNodes.put(id, instances);
+ synchronized (this) {
+ // recheck in the synchronized block and recreate
+ if (recentNodes.get(id) == null) {
+ recentNodes.put(id, new LinkedList<NodeInstance>());
+ }
+ instances = recentNodes.get(id);
+ }
}
return instances;
}
-
+
/**
- * Sort an available node list
+ * Sort a the recent node list for a single role
* @param role role to sort
*/
- private void sortAvailableNodeList(int role) {
- List<NodeInstance> nodesForRoleId = getNodesForRoleId(role);
+ private void sortRecentNodeList(int role) {
+ List<NodeInstance> nodesForRoleId = getRecentNodesForRoleId(role);
if (nodesForRoleId != null) {
Collections.sort(nodesForRoleId, new NodeInstance.Preferred(role));
}
@@ -566,7 +570,7 @@ public class RoleHistory {
NodeInstance nodeInstance = null;
// Get the list of possible targets.
// This is a live list: changes here are preserved
- List<NodeInstance> targets = getNodesForRoleId(roleId);
+ List<NodeInstance> targets = getRecentNodesForRoleId(roleId);
if (targets == null) {
// nothing to allocate on
return null;
@@ -655,7 +659,7 @@ public class RoleHistory {
public synchronized List<NodeInstance> listActiveNodes(int role) {
return nodemap.listActiveNodes(role);
}
-
+
/**
* Get the node entry of a container
* @param container container to look up
@@ -705,7 +709,7 @@ public class RoleHistory {
* @return list of containers potentially reordered
*/
public synchronized List<Container> prepareAllocationList(List<Container> allocatedContainers) {
-
+
//partition into requested and unrequested
List<Container> requested =
new ArrayList<>(allocatedContainers.size());
@@ -717,7 +721,7 @@ public class RoleHistory {
requested.addAll(unrequested);
return requested;
}
-
+
/**
* A container has been allocated on a node -update the data structures
* @param container container
@@ -730,7 +734,7 @@ public class RoleHistory {
int actualCount) {
int role = ContainerPriority.extractRole(container);
String hostname = RoleHistoryUtils.hostnameOf(container);
- List<NodeInstance> nodeInstances = getOrCreateNodesForRoleId(role);
+ List<NodeInstance> nodeInstances = listRecentNodesForRoleId(role);
ContainerAllocation outcome =
outstandingRequests.onContainerAllocated(role, hostname, container);
if (desiredCount <= actualCount) {
@@ -741,7 +745,7 @@ public class RoleHistory {
//add the list
log.info("Adding {} hosts for role {}", hosts.size(), role);
nodeInstances.addAll(hosts);
- sortAvailableNodeList(role);
+ sortRecentNodeList(role);
}
}
return outcome;
@@ -892,7 +896,7 @@ public class RoleHistory {
/**
* If the node is marked as available; queue it for assignments.
- * Unsynced: expects caller to be in a sync block.
+ * Unsynced: requires caller to be in a sync block.
* @param container completed container
* @param nodeEntry node
* @param available available flag
@@ -907,7 +911,7 @@ public class RoleHistory {
NodeInstance ni = getOrCreateNodeInstance(container);
int roleId = ContainerPriority.extractRole(container);
log.debug("Node {} is now available for role id {}", ni, roleId);
- getOrCreateNodesForRoleId(roleId).addFirst(ni);
+ listRecentNodesForRoleId(roleId).addFirst(ni);
}
return available;
}
@@ -918,8 +922,7 @@ public class RoleHistory {
public synchronized void dump() {
for (ProviderRole role : providerRoles) {
log.info(role.toString());
- List<NodeInstance> instances =
- getOrCreateNodesForRoleId(role.id);
+ List<NodeInstance> instances = listRecentNodesForRoleId(role.id);
log.info(" available: " + instances.size()
+ " " + SliderUtils.joinWithInnerSeparator(" ", instances));
}
@@ -952,7 +955,7 @@ public class RoleHistory {
*/
@VisibleForTesting
public List<NodeInstance> cloneAvailableList(int role) {
- return new LinkedList<>(getOrCreateNodesForRoleId(role));
+ return new LinkedList<>(listRecentNodesForRoleId(role));
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e9a95b9/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy
index aa7bb11..c62eb72 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy
@@ -34,6 +34,7 @@ import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.RoleInstance
import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
@@ -47,11 +48,6 @@ import org.junit.Test
class TestMockAppStateDynamicHistory extends BaseMockAppStateTest
implements MockRoles {
- @Override
- String getTestName() {
- return "TestMockAppStateDynamicHistory"
- }
-
/**
* Small cluster with multiple containers per node,
* to guarantee many container allocations on each node
@@ -62,26 +58,6 @@ class TestMockAppStateDynamicHistory extends BaseMockAppStateTest
return new MockYarnEngine(8, 1)
}
- @Override
- void initApp() {
- super.initApp()
- appState = new MockAppState()
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
-
- def instance = factory.newInstanceDefinition(0,0,0)
-
- appState.buildInstance(
- instance,
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- null,
- null, new SimpleReleaseSelector())
- }
-
-
@Test
public void testDynamicRoleHistory() throws Throwable {
@@ -199,7 +175,7 @@ class TestMockAppStateDynamicHistory extends BaseMockAppStateTest
assert !entry.live
- def nodesForRoleId = roleHistory.getNodesForRoleId(role_priority_8)
+ def nodesForRoleId = roleHistory.getRecentNodesForRoleId(role_priority_8)
assert nodesForRoleId
// make sure new nodes will default to a different host in the engine
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e9a95b9/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
index ee4abd6..e35f028 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
@@ -23,6 +23,7 @@ import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.slider.api.ResourceKeys
+import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockAppState
@@ -31,6 +32,7 @@ import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo
import org.apache.slider.server.appmaster.state.ContainerPriority
import org.apache.slider.server.appmaster.state.RoleHistoryUtils
import org.apache.slider.server.appmaster.state.RoleInstance
@@ -65,40 +67,27 @@ class TestMockAppStateDynamicRoles extends BaseMockAppStateTest
}
@Override
- void initApp() {
- super.initApp()
- appState = new MockAppState()
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
- def instance = factory.newInstanceDefinition(0,0,0)
-
+ AggregateConf buildInstanceDefinition() {
+ def instance = factory.newInstanceDefinition(0, 0, 0)
def opts = [
- (ResourceKeys.COMPONENT_PRIORITY): ROLE4,
+ (ResourceKeys.COMPONENT_PRIORITY) : ROLE4,
(ResourceKeys.COMPONENT_INSTANCES): "1",
]
- instance.resourceOperations.components[ROLE4]= opts
+ instance.resourceOperations.components[ROLE4] = opts
def opts5 = [
- (ResourceKeys.COMPONENT_PRIORITY) : ROLE5,
- (ResourceKeys.COMPONENT_INSTANCES): "1",
+ (ResourceKeys.COMPONENT_PRIORITY) : ROLE5,
+ (ResourceKeys.COMPONENT_INSTANCES) : "1",
(ResourceKeys.COMPONENT_PLACEMENT_POLICY):
Integer.toString(PlacementPolicy.STRICT),
- (ResourceKeys.NODE_FAILURE_THRESHOLD):
+ (ResourceKeys.NODE_FAILURE_THRESHOLD) :
Integer.toString(2),
]
- instance.resourceOperations.components[ROLE5]= opts5
-
- appState.buildInstance(
- instance,
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- null,
- null, new SimpleReleaseSelector())
+ instance.resourceOperations.components[ROLE5] = opts5
+ instance
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e9a95b9/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
index 5d880b4..7bc6fe4 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
@@ -20,21 +20,22 @@ package org.apache.slider.server.appmaster.model.appstate
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.slider.api.ResourceKeys
+import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.core.exceptions.BadConfigException
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo
import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector
import org.apache.slider.server.avro.RoleHistoryWriter
import org.junit.Test
/**
- * Test that if you have >1 role, the right roles are chosen for release.
+ * Test that if you have more than one role, the right roles are chosen for release.
*/
@CompileStatic
@Slf4j
@@ -57,32 +58,25 @@ class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
}
@Override
- void initApp() {
- super.initApp()
- appState = new MockAppState()
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
+ AppStateBindingInfo buildBindingInfo() {
+ def bindingInfo = super.buildBindingInfo()
+ bindingInfo.releaseSelector = new MostRecentContainerReleaseSelector()
+ bindingInfo
+ }
+ @Override
+ AggregateConf buildInstanceDefinition() {
def instance = factory.newInstanceDefinition(0, 0, 0)
def opts = [
(ResourceKeys.COMPONENT_INSTANCES): "1",
- (ResourceKeys.COMPONENT_PRIORITY): "6",
+ (ResourceKeys.COMPONENT_PRIORITY) : "6",
]
instance.resourceOperations.components["dynamic-6"] = opts
-
-
- appState.buildInstance(instance,
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- null, null,
- new MostRecentContainerReleaseSelector())
+ instance
}
-
private ConfTreeOperations init() {
createAndStartNodes();
def resources = appState.instanceDefinition.resources;
@@ -181,16 +175,10 @@ class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
def historyWorkDir2 = new File("target/history" + testName + "-0002")
def historyPath2 = new Path(historyWorkDir2.toURI())
appState = new MockAppState()
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
- appState.buildInstance(
- factory.newInstanceDefinition(0, 0, 0),
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath2,
- null, null,
- new MostRecentContainerReleaseSelector())
+ def binding2 = buildBindingInfo()
+ binding2.instanceDefinition = factory.newInstanceDefinition(0, 0, 0)
+ binding2.historyPath = historyPath2
+ appState.buildInstance(binding2)
// on this read there won't be the right number of roles
def loadedRoleHistory = historyWriter.read(fs, history)
assert 0 == appState.roleHistory.rebuild(loadedRoleHistory)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e9a95b9/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
index c310583..02052c2 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
@@ -20,14 +20,11 @@ package org.apache.slider.server.appmaster.model.appstate
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.yarn.api.records.Container
import org.apache.slider.api.StatusKeys
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoles
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation
-import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector
import org.apache.slider.server.appmaster.state.NodeEntry
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.NodeMap
@@ -35,7 +32,7 @@ import org.apache.slider.server.appmaster.state.RoleInstance
import org.junit.Test
/**
- * Test that if you have >1 role, the right roles are chosen for release.
+ * Test that app state is rebuilt on a restart
*/
@CompileStatic
@Slf4j
@@ -74,16 +71,11 @@ class TestMockAppStateRebuildOnAMRestart extends BaseMockAppStateTest
appState = new MockAppState()
//and rebuild
- appState.buildInstance(
- factory.newInstanceDefinition(r0, r1, r2),
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- containers,
- null,
- new MostRecentContainerReleaseSelector())
+
+ def bindingInfo = buildBindingInfo()
+ bindingInfo.instanceDefinition = factory.newInstanceDefinition(r0, r1, r2)
+ bindingInfo.liveContainers = containers
+ appState.buildInstance(bindingInfo)
assert appState.startedCountainerCount == clusterSize
@@ -107,22 +99,18 @@ class TestMockAppStateRebuildOnAMRestart extends BaseMockAppStateTest
assertNotNull("Null entry in original nodemap for " + hostname, orig)
for (int i = 0; i < ROLE_COUNT; i++) {
-
- assert (nodeInstance.getActiveRoleInstances(i) ==
- orig.getActiveRoleInstances(i))
+ assert (nodeInstance.getActiveRoleInstances(i) == orig.getActiveRoleInstances(i))
NodeEntry origRE = orig.getOrCreate(i)
NodeEntry newRE = nodeInstance.getOrCreate(i)
assert origRE.live == newRE.live
- assert newRE.starting == 0
+ assert 0 == newRE.starting
}
}
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- assert ops.size() == 0
+ assert 0 == appState.reviewRequestAndReleaseNodes().size()
def status = appState.getClusterStatus()
// verify the AM restart container count was set
- String restarted = status.getInfo(
- StatusKeys.INFO_CONTAINERS_AM_RESTART)
+ String restarted = status.getInfo(StatusKeys.INFO_CONTAINERS_AM_RESTART)
assert restarted != null;
//and that the count == 1 master + the region servers
assert Integer.parseInt(restarted) == containers.size()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e9a95b9/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
index 4ba0afd..ad607cf 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
@@ -25,6 +25,7 @@ import org.apache.slider.api.ResourceKeys
import org.apache.slider.core.conf.ConfTree
import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
@@ -71,7 +72,7 @@ class TestMockContainerResourceAllocations extends BaseMockAppStateTest {
assert ops.size() == 1
ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
Resource requirements = operation.request.capability
- assert requirements.memory == RM_MAX_RAM
+ assert requirements.memory == MockAppState.RM_MAX_RAM
assert requirements.virtualCores == 2
}
@@ -89,7 +90,7 @@ class TestMockContainerResourceAllocations extends BaseMockAppStateTest {
ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
Resource requirements = operation.request.capability
assert requirements.memory == 512
- assert requirements.virtualCores == RM_MAX_CORES
+ assert requirements.virtualCores == MockAppState.RM_MAX_CORES
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e9a95b9/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
index 4fdf39c..f4eb4ae 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
@@ -68,7 +68,7 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
public void setupRH() {
roleHistory.onStart(fs, historyPath)
roleHistory.insert(nodes)
- roleHistory.buildAvailableNodeLists();
+ roleHistory.buildRecentNodeLists();
resource = Resource.newInstance(ResourceKeys.DEF_YARN_CORES,
ResourceKeys.DEF_YARN_MEMORY);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e9a95b9/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy
index 79d23e5..c4768ec 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy
@@ -63,7 +63,7 @@ class TestRoleHistoryFindNodesForNewInstances extends BaseMockAppStateTest {
@Before
public void setupNodeMap() {
roleHistory.insert(nodes)
- roleHistory.buildAvailableNodeLists();
+ roleHistory.buildRecentNodeLists();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e9a95b9/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
index c81c686..7afcfc1 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
@@ -26,11 +26,9 @@ import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.state.NodeEntry
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.RoleHistory
-import org.apache.slider.server.avro.LoadedRoleHistory
import org.apache.slider.server.avro.RoleHistoryWriter
import org.junit.Test
@@ -136,7 +134,7 @@ class TestRoleHistoryRW extends BaseMockAppStateTest {
assert rh2.thawedDataTime == savetime
// now start it
- rh2.buildAvailableNodeLists();
+ rh2.buildRecentNodeLists();
describe("starting")
rh2.dump();
List<NodeInstance> available0 = rh2.cloneAvailableList(0)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e9a95b9/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
index 9847992..db795d0 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
@@ -67,7 +67,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
@Before
public void setupNodeMap() {
roleHistory.insert(nodes)
- roleHistory.buildAvailableNodeLists();
+ roleHistory.buildRecentNodeLists();
}
@Test
@@ -141,7 +141,7 @@ class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
recordAsFailed(age4Active1, key, 4)
// trigger a list rebuild
- roleHistory.buildAvailableNodeLists();
+ roleHistory.buildRecentNodeLists();
assert !roleHistory.cloneAvailableList(key).isEmpty()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e9a95b9/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
index 29eefa5..33ea0a0 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
@@ -20,7 +20,6 @@ package org.apache.slider.server.appmaster.model.mock
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.Path
import org.apache.hadoop.yarn.api.records.Container
@@ -34,21 +33,19 @@ import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo
import org.apache.slider.server.appmaster.state.ContainerAssignment
import org.apache.slider.server.appmaster.state.ContainerOutcome
import org.apache.slider.server.appmaster.state.NodeEntry
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.RoleInstance
import org.apache.slider.server.appmaster.state.RoleStatus
-import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
import org.apache.slider.test.SliderTestBase
import org.junit.Before
@CompileStatic
@Slf4j
abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles {
- public static final int RM_MAX_RAM = 4096
- public static final int RM_MAX_CORES = 64
MockFactory factory = new MockFactory()
MockAppState appState
MockYarnEngine engine
@@ -77,12 +74,14 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
return new MockYarnEngine(64, 1)
}
+ /**
+ * Initialize the application.
+ * This uses the binding information supplied by {@link #buildBindingInfo()}.
+ */
@Before
void initApp(){
String historyDirName = testName;
-
-
YarnConfiguration conf = SliderUtils.createConfiguration()
applicationId = new MockApplicationId(id: 1, clusterTimestamp: 0)
applicationAttemptId = new MockApplicationAttemptId(
@@ -94,29 +93,35 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
historyPath = new Path(historyWorkDir.toURI())
fs.delete(historyPath, true)
appState = new MockAppState()
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
- appState.buildInstance(
- buildInstanceDefinition(),
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- null, null,
- new SimpleReleaseSelector())
+ appState.buildInstance(buildBindingInfo())
}
/**
- * Override point, define the instance definition
+ * Build the binding info from the default constructor values,
+ * the roles from {@link #factory}, and an instance definition
+ * from {@link #buildInstanceDefinition()}
* @return
*/
+ AppStateBindingInfo buildBindingInfo() {
+ AppStateBindingInfo binding = new AppStateBindingInfo()
+ binding.instanceDefinition = buildInstanceDefinition();
+ binding.roles = factory.ROLES
+ binding.fs = fs
+ binding.historyPath = historyPath
+ binding
+ }
+
+ /**
+ * Override point, define the instance definition
+ * @return the instance definition
+ */
public AggregateConf buildInstanceDefinition() {
factory.newInstanceDefinition(0, 0, 0)
}
/**
* Get the test name ... defaults to method name
- * @return
+ * @return the method name
*/
String getTestName() {
methodName.methodName;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e9a95b9/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
index 6e21a38..5565e6b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
@@ -21,12 +21,14 @@ import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring
import org.apache.slider.server.appmaster.state.AbstractClusterServices
import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo
/**
* Extended app state that makes more things public
*/
class MockAppState extends AppState {
-
+ public static final int RM_MAX_RAM = 4096
+ public static final int RM_MAX_CORES = 64
public MockAppState(AbstractClusterServices recordFactory) {
super(recordFactory, new MetricsAndMonitoring());
}
@@ -38,6 +40,12 @@ class MockAppState extends AppState {
*/
public MockAppState() {
super(new MockClusterServices(), new MetricsAndMonitoring());
+ setContainerLimits(1, RM_MAX_RAM, 1, RM_MAX_CORES)
+ }
+
+ MockAppState(AppStateBindingInfo bindingInfo) {
+ this()
+ buildInstance(bindingInfo)
}
public Map<String, ProviderRole> getRoleMap() {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e9a95b9/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
index 4c43168..7237ff4 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
@@ -24,10 +24,10 @@ import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.api.json.JSONConfiguration;
-import junit.framework.Assert;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.common.SliderKeys;
@@ -35,10 +35,12 @@ import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
+import org.apache.slider.server.appmaster.model.mock.MockAppState;
import org.apache.slider.server.appmaster.model.mock.MockFactory;
import org.apache.slider.server.appmaster.model.mock.MockProviderService;
import org.apache.slider.server.appmaster.model.mock.MockClusterServices;
import org.apache.slider.server.appmaster.state.AppState;
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
import org.apache.slider.server.appmaster.state.ProviderAppState;
import org.apache.slider.server.appmaster.state.SimpleReleaseSelector;
import org.apache.slider.server.appmaster.web.WebAppApi;
@@ -71,26 +73,19 @@ public class TestAMAgentWebServices {
public boolean verify(String hostname,
javax.net.ssl.SSLSession sslSession) {
- if (hostname.equals("localhost")) {
- return true;
- }
- return false;
+ return hostname.equals("localhost");
}
});
-
}
protected static final Logger log =
LoggerFactory.getLogger(TestAMAgentWebServices.class);
- public static final int RM_MAX_RAM = 4096;
- public static final int RM_MAX_CORES = 64;
public static final String AGENT_URL =
"https://localhost:${PORT}/ws/v1/slider/agents/";
static MockFactory factory = new MockFactory();
- private static Configuration conf = new Configuration();
private static WebAppApi slider;
private static FileSystem fs;
@@ -117,28 +112,16 @@ public class TestAMAgentWebServices {
YarnConfiguration conf = SliderUtils.createConfiguration();
fs = FileSystem.get(new URI("file:///"), conf);
AppState appState = null;
- try {
- fs = FileSystem.get(new URI("file:///"), conf);
- File
- historyWorkDir =
- new File("target/history", "TestAMAgentWebServices");
- org.apache.hadoop.fs.Path
- historyPath =
- new org.apache.hadoop.fs.Path(historyWorkDir.toURI());
- fs.delete(historyPath, true);
- appState = new AppState(new MockClusterServices(), new MetricsAndMonitoring());
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES);
- appState.buildInstance(
- factory.newInstanceDefinition(0, 0, 0),
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- null, null, new SimpleReleaseSelector());
- } catch (Exception e) {
- log.error("Failed to set up app {}", e, e);
- }
+ File historyWorkDir = new File("target/history", "TestAMAgentWebServices");
+ Path historyPath = new Path(historyWorkDir.toURI());
+ fs.delete(historyPath, true);
+ appState = new MockAppState(new MockClusterServices());
+ AppStateBindingInfo binding = new AppStateBindingInfo();
+ binding.instanceDefinition = factory.newInstanceDefinition(0, 0, 0);
+ binding.roles = MockFactory.ROLES;
+ binding.fs = fs;
+ binding.historyPath = historyPath;
+ appState.buildInstance(binding);
ProviderAppState providerAppState = new ProviderAppState("undefined",
appState);
@@ -173,7 +156,7 @@ public class TestAMAgentWebServices {
WebResource webResource = client.resource(base_url + "test/register");
response = webResource.type(MediaType.APPLICATION_JSON)
.post(RegistrationResponse.class, createDummyJSONRegister());
- Assert.assertEquals(RegistrationStatus.OK, response.getResponseStatus());
+ assertEquals(RegistrationStatus.OK, response.getResponseStatus());
}
protected Client createTestClient() {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4e9a95b9/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
index df7e002..70a6bcb 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
@@ -31,7 +31,6 @@ import com.sun.jersey.test.framework.JerseyTest;
import com.sun.jersey.test.framework.WebAppDescriptor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.AggregateConf;
@@ -39,13 +38,13 @@ import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.exceptions.BadClusterStateException;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.core.persist.JsonSerDeser;
-import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
+import org.apache.slider.server.appmaster.model.mock.MockAppState;
+import org.apache.slider.server.appmaster.model.mock.MockClusterServices;
import org.apache.slider.server.appmaster.model.mock.MockFactory;
import org.apache.slider.server.appmaster.model.mock.MockProviderService;
-import org.apache.slider.server.appmaster.model.mock.MockClusterServices;
import org.apache.slider.server.appmaster.state.AppState;
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
import org.apache.slider.server.appmaster.state.ProviderAppState;
-import org.apache.slider.server.appmaster.state.SimpleReleaseSelector;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.WebAppApiImpl;
import org.apache.slider.server.appmaster.web.rest.AMWebServices;
@@ -67,15 +66,11 @@ import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
public class TestAMManagementWebServices extends JerseyTest {
protected static final Logger log =
LoggerFactory.getLogger(TestAMManagementWebServices.class);
- public static final int RM_MAX_RAM = 4096;
- public static final int RM_MAX_CORES = 64;
- public static final String EXAMPLES =
- "/org/apache/slider/core/conf/examples/";
+ public static final String EXAMPLES = "/org/apache/slider/core/conf/examples/";
static MockFactory factory = new MockFactory();
private static Configuration conf = new Configuration();
private static WebAppApi slider;
@@ -116,40 +111,26 @@ public class TestAMManagementWebServices extends JerseyTest {
}
protected AggregateConf getAggregateConf() {
- JsonSerDeser<ConfTree> confTreeJsonSerDeser =
- new JsonSerDeser<ConfTree>(ConfTree.class);
- ConfTree internal = null;
- ConfTree app_conf = null;
- ConfTree resources = null;
try {
- internal =
- confTreeJsonSerDeser.fromResource(
- EXAMPLES +"internal.json");
- app_conf =
- confTreeJsonSerDeser.fromResource(
- EXAMPLES + "app_configuration.json");
- resources =
- confTreeJsonSerDeser.fromResource(
- EXAMPLES + "resources.json");
+ JsonSerDeser<ConfTree> confTreeJsonSerDeser = new JsonSerDeser<>(ConfTree.class);
+ AggregateConf aggregateConf = new AggregateConf(
+ confTreeJsonSerDeser.fromResource(EXAMPLES + "internal.json"),
+ confTreeJsonSerDeser.fromResource(EXAMPLES + "app_configuration.json"),
+ confTreeJsonSerDeser.fromResource(EXAMPLES + "resources.json"));
+ aggregateConf.setName("test");
+ return aggregateConf;
} catch (IOException e) {
- fail(e.getMessage());
+ throw new AssertionError(e.getMessage(), e);
}
- AggregateConf aggregateConf = new AggregateConf(
- resources,
- app_conf,
- internal);
- aggregateConf.setName("test");
- return aggregateConf;
}
-
}
+
@Before
@Override
public void setUp() throws Exception {
super.setUp();
injector = createInjector();
- YarnConfiguration conf = SliderUtils.createConfiguration();
- fs = FileSystem.get(new URI("file:///"), conf);
+ fs = FileSystem.get(new URI("file:///"), SliderUtils.createConfiguration());
}
private static Injector createInjector() {
@@ -160,23 +141,17 @@ public class TestAMManagementWebServices extends JerseyTest {
AppState appState = null;
try {
fs = FileSystem.get(new URI("file:///"), conf);
- File
- historyWorkDir =
- new File("target/history", "TestAMManagementWebServices");
- org.apache.hadoop.fs.Path
- historyPath =
+ File historyWorkDir = new File("target/history", "TestAMManagementWebServices");
+ org.apache.hadoop.fs.Path historyPath =
new org.apache.hadoop.fs.Path(historyWorkDir.toURI());
fs.delete(historyPath, true);
- appState = new AppState(new MockClusterServices(), new MetricsAndMonitoring());
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES);
- appState.buildInstance(
- factory.newInstanceDefinition(0, 0, 0),
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- null, null, new SimpleReleaseSelector());
+ appState = new MockAppState(new MockClusterServices());
+ AppStateBindingInfo binding = new AppStateBindingInfo();
+ binding.instanceDefinition = factory.newInstanceDefinition(0, 0, 0);
+ binding.roles = MockFactory.ROLES;
+ binding.fs = fs;
+ binding.historyPath = historyPath;
+ appState.buildInstance(binding);
} catch (IOException | BadClusterStateException | URISyntaxException | BadConfigException e) {
log.error("{}", e, e);
}