You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/07/21 18:39:12 UTC
[10/52] [abbrv] hadoop git commit: YARN-6255. Refactor
yarn-native-services framework. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c242ac2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
deleted file mode 100644
index 9eb7d5c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.persist;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.io.Files;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.Path;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
-import org.apache.slider.common.params.Arguments;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.providers.agent.AgentKeys;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Class to prepare and persist app and add-on definitions.
- *
- * In this case, the app definition and add-on definitions are auto-inferred from the user input rather than explicit
- * inclusion of application package in the config.
- *
- * Processing an app definition involves one or more of the following: - modify appConfig - package definition into a
- * temporary folder - upload to HDFS
- *
- * This class keeps track of all the required operations and allows them to be invoked by build operation
- */
-public class AppDefinitionPersister {
- private static final Logger log =
- LoggerFactory.getLogger(AppDefinitionPersister.class);
-
- private final SliderFileSystem sliderFileSystem;
- private List<AppDefinition> appDefinitions;
-
- public AppDefinitionPersister(SliderFileSystem sliderFileSystem) {
- this.sliderFileSystem = sliderFileSystem;
- appDefinitions = new ArrayList<>();
- }
-
-
- /**
- * Process the application package or folder by copying it to the cluster path
- *
- * @param appDefinition details of application package
- *
- * @throws BadConfigException
- * @throws IOException
- */
- private void persistDefinitionPackageOrFolder(AppDefinition appDefinition)
- throws BadConfigException, IOException {
- if (!appDefinition.appDefPkgOrFolder.canRead()) {
- throw new BadConfigException("Pkg/Folder cannot be accessed - "
- + appDefinition.appDefPkgOrFolder.getAbsolutePath());
- }
-
- File src = appDefinition.appDefPkgOrFolder;
- String targetName = appDefinition.pkgName;
- log.debug("Package name: " + targetName);
- if (appDefinition.appDefPkgOrFolder.isDirectory()) {
- log.info("Processing app package/folder {} for {}",
- appDefinition.appDefPkgOrFolder.getAbsolutePath(),
- appDefinition.pkgName);
- File tmpDir = Files.createTempDir();
- File zipFile = new File(tmpDir.getCanonicalPath(), File.separator + appDefinition.pkgName);
- SliderUtils.zipFolder(appDefinition.appDefPkgOrFolder, zipFile);
- src = zipFile;
- }
-
- sliderFileSystem.getFileSystem().copyFromLocalFile(
- false,
- false,
- new Path(src.toURI()),
- new Path(appDefinition.targetFolderInFs, targetName));
- }
-
- public void persistPackages() throws BadConfigException, IOException {
- for (AppDefinition appDefinition : appDefinitions) {
- persistDefinitionPackageOrFolder(appDefinition);
- }
- }
-
- public void processSuppliedDefinitions(String clustername,
- AbstractClusterBuildingActionArgs buildInfo,
- ConfTreeOperations appConf)
- throws BadConfigException, IOException, BadCommandArgumentsException {
- // if metainfo is provided add to the app instance
- if (buildInfo.appMetaInfo != null || buildInfo.appMetaInfoJson != null) {
- if (buildInfo.appMetaInfo != null && buildInfo.appMetaInfoJson != null) {
- throw new BadConfigException("Both %s and %s cannot be specified",
- Arguments.ARG_METAINFO, Arguments.ARG_METAINFO_JSON);
- }
-
- // Now we know that only one of either file or JSON is used
- boolean isFileUsed = buildInfo.appMetaInfo != null ? true : false;
- String argUsed = isFileUsed ? Arguments.ARG_METAINFO
- : Arguments.ARG_METAINFO_JSON;
-
- if (buildInfo.appDef != null) {
- throw new BadConfigException("Both %s and %s cannot be specified",
- argUsed, Arguments.ARG_APPDEF);
- }
- if (SliderUtils.isSet(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) {
- throw new BadConfigException(
- "%s cannot not be set if %s is specified in the cmd line ",
- AgentKeys.APP_DEF, argUsed);
- }
-
- if (isFileUsed) {
- if (!buildInfo.appMetaInfo.canRead() || !buildInfo.appMetaInfo.isFile()) {
- throw new BadConfigException(
- "Path specified with %s either cannot be read or is not a file",
- Arguments.ARG_METAINFO);
- }
- } else {
- if (StringUtils.isEmpty(buildInfo.appMetaInfoJson.trim())) {
- throw new BadConfigException("Empty string specified with %s",
- Arguments.ARG_METAINFO_JSON);
- }
- }
-
- File tempDir = Files.createTempDir();
- File pkgSrcDir = new File(tempDir, "default");
- if (!pkgSrcDir.exists() && !pkgSrcDir.mkdirs()) {
- throw new IOException("Failed to create directory: " + pkgSrcDir);
- }
- File destMetaInfo = new File(pkgSrcDir, "metainfo.json");
- if (isFileUsed) {
- if (buildInfo.appMetaInfo.getName().endsWith(".xml")) {
- Files.copy(buildInfo.appMetaInfo, new File(pkgSrcDir, "metainfo.xml"));
- } else {
- Files.copy(buildInfo.appMetaInfo, destMetaInfo);
- }
- } else {
- Files.write(
- buildInfo.appMetaInfoJson.getBytes(Charset.forName("UTF-8")),
- destMetaInfo);
- }
-
- Path appDirPath = sliderFileSystem.buildAppDefDirPath(clustername);
- log.info("Using default app def path {}", appDirPath.toString());
-
- appDefinitions.add(new AppDefinition(appDirPath, pkgSrcDir,
- SliderKeys.DEFAULT_APP_PKG));
- Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG);
- appConf.getGlobalOptions().set(AgentKeys.APP_DEF, appDefPath);
- log.info("Setting app package to {}.", appDefPath);
- }
-
- if (buildInfo.appDef != null) {
- if (SliderUtils.isSet(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) {
- throw new BadConfigException("application.def must not be set if --appdef is provided.");
- }
-
- if (!buildInfo.appDef.exists()) {
- throw new BadConfigException("--appdef is not a valid path.");
- }
-
- Path appDirPath = sliderFileSystem.buildAppDefDirPath(clustername);
- appDefinitions.add(new AppDefinition(appDirPath, buildInfo.appDef, SliderKeys.DEFAULT_APP_PKG));
- Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG);
- appConf.getGlobalOptions().set(AgentKeys.APP_DEF, appDefPath);
- log.info("Setting app package to {}.", appDefPath);
- }
-
- if (buildInfo.addonDelegate.getAddonMap().size() > 0) {
- if (SliderUtils.isUnset(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) {
- throw new BadConfigException("addon package can only be specified if main app package is specified.");
- }
-
- List<String> addons = new ArrayList<String>();
- Map<String, String> addonMap = buildInfo.addonDelegate.getAddonMap();
- for (Map.Entry<String, String > entry : addonMap.entrySet()) {
- String key = entry.getKey();
- String value = entry.getValue();
- if (SliderUtils.isUnset(value)) {
- throw new BadConfigException("Invalid path for addon package " + key);
- }
- File defPath = new File(value);
- if (!defPath.exists()) {
- throw new BadConfigException("addon folder or package path is not valid.");
- }
-
- Path addonPath = sliderFileSystem.buildAddonDirPath(clustername, key);
- String addonPkgName = "addon_" + key + ".zip";
-
- log.debug(
- "addonMap.get(key): {} addonPath: {} defPath: {} addonPkgName: {}",
- addonMap.get(key), addonPath, defPath, addonPkgName);
-
- appDefinitions.add(new AppDefinition(addonPath, defPath, addonPkgName));
- String addOnKey = AgentKeys.ADDON_PREFIX + key;
- Path addonPkgPath = new Path(addonPath, addonPkgName);
- log.info("Setting addon package {} to {}.", addOnKey, addonPkgPath);
- appConf.getGlobalOptions().set(addOnKey, addonPkgPath);
- addons.add(addOnKey);
- }
-
- String existingList = appConf.getGlobalOptions().get(AgentKeys.ADDONS);
- if (SliderUtils.isUnset(existingList)) {
- existingList = "";
- }
- appConf.getGlobalOptions().set(AgentKeys.ADDONS, existingList + StringUtils.join(addons, ","));
- }
- }
-
-
- @VisibleForTesting
- public List<AppDefinitionPersister.AppDefinition> getAppDefinitions() {
- return appDefinitions;
- }
-
- // Helper class to hold details for the app and addon packages
- static class AppDefinition {
- // The target folder where the package will be stored
- public Path targetFolderInFs;
- // The on disk location of the app def package or folder
- public File appDefPkgOrFolder;
- // Package name
- public String pkgName;
-
- public AppDefinition(Path targetFolderInFs, File appDefPkgOrFolder, String pkgName) {
- this.targetFolderInFs = targetFolderInFs;
- this.appDefPkgOrFolder = appDefPkgOrFolder;
- this.pkgName = pkgName;
- }
-
- @Override
- public String toString() {
- return new StringBuilder().append("targetFolderInFs").append(" : ").append(targetFolderInFs.toString())
- .append(", ")
- .append("appDefPkgOrFolder").append(" : ").append(appDefPkgOrFolder.toString())
- .append(", ")
- .append("pkgName").append(" : ").append(pkgName).toString();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c242ac2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
index 4f60c06..8fe2549 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
@@ -29,6 +29,7 @@ import org.codehaus.jackson.JsonParseException;
import org.codehaus.jackson.map.DeserializationConfig;
import org.codehaus.jackson.map.JsonMappingException;
import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.PropertyNamingStrategy;
import org.codehaus.jackson.map.SerializationConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,6 +66,11 @@ public class JsonSerDeser<T> {
mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
}
+ public JsonSerDeser(Class<T> classType, PropertyNamingStrategy namingStrategy) {
+ this(classType);
+ mapper.setPropertyNamingStrategy(namingStrategy);
+ }
+
/**
* Convert from JSON
* @param json input
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c242ac2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
index 01444fd..42e103a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
@@ -103,19 +103,6 @@ public abstract class AbstractClientProvider extends Configured {
/**
- * Any provider-side alteration of a configuration can take place here.
- * @param aggregateConf config to patch
- * @throws IOException IO problems
- * @throws SliderException Slider-specific issues
- */
- public void prepareInstanceConfiguration(AggregateConf aggregateConf) throws
- SliderException,
- IOException {
- //default: do nothing
- }
-
-
- /**
* Prepare the AM settings for launch
* @param fileSystem filesystem
* @param serviceConf configuration of the client
@@ -234,7 +221,7 @@ public abstract class AbstractClientProvider extends Configured {
* @param appDescription brief description of the application
* @return
*/
- public final Set<String> createApplicationTags(String appName,
+ public static final Set<String> createApplicationTags(String appName,
String appVersion, String appDescription) {
Set<String> tags = new HashSet<>();
tags.add(SliderUtils.createNameTag(appName));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c242ac2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
deleted file mode 100644
index 41b26e9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ /dev/null
@@ -1,438 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
-import org.apache.hadoop.registry.client.types.AddressTypes;
-import org.apache.hadoop.registry.client.types.Endpoint;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.slider.api.ClusterDescription;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.tools.ConfigHelper;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.main.ExitCodeProvider;
-import org.apache.slider.server.appmaster.actions.QueueAccess;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
-import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.services.workflow.ForkedProcessService;
-import org.apache.slider.server.services.workflow.ServiceParent;
-import org.apache.slider.server.services.workflow.WorkflowSequenceService;
-import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-
-/**
- * The base class for provider services. It lets the implementations
- * add sequences of operations, and propagates service failures
- * upstream
- */
-public abstract class AbstractProviderService
- extends WorkflowSequenceService
- implements
- ProviderCore,
- SliderKeys,
- ProviderService {
- private static final Logger log =
- LoggerFactory.getLogger(AbstractProviderService.class);
- protected StateAccessForProviders amState;
- protected URL amWebAPI;
- protected YarnRegistryViewForProviders yarnRegistry;
- protected QueueAccess queueAccess;
-
- protected AbstractProviderService(String name) {
- super(name);
- setStopIfNoChildServicesAtStartup(false);
- }
-
- @Override
- public Configuration getConf() {
- return getConfig();
- }
-
- public StateAccessForProviders getAmState() {
- return amState;
- }
-
- public QueueAccess getQueueAccess() {
- return queueAccess;
- }
-
- public void setAmState(StateAccessForProviders amState) {
- this.amState = amState;
- }
-
- @Override
- public String getHumanName() {
- return getName().toLowerCase(Locale.ENGLISH);
- }
-
- @Override
- public void bind(StateAccessForProviders stateAccessor,
- QueueAccess queueAccess,
- List<Container> liveContainers) {
- this.amState = stateAccessor;
- this.queueAccess = queueAccess;
- }
-
- @Override
- public void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry) {
- this.yarnRegistry = yarnRegistry;
- }
-
- public YarnRegistryViewForProviders getYarnRegistry() {
- return yarnRegistry;
- }
-
- @Override
- public void notifyContainerCompleted(ContainerId containerId) {
- }
-
- /**
- * Load default Configuration
- * @param confDir configuration directory
- * @return configuration
- * @throws BadCommandArgumentsException
- * @throws IOException
- */
- @Override
- public Configuration loadProviderConfigurationInformation(File confDir)
- throws BadCommandArgumentsException, IOException {
- return new Configuration(false);
- }
-
- /**
- * Load a specific XML configuration file for the provider config
- * @param confDir configuration directory
- * @param siteXMLFilename provider-specific filename
- * @return a configuration to be included in status
- * @throws BadCommandArgumentsException argument problems
- * @throws IOException IO problems
- */
- protected Configuration loadProviderConfigurationInformation(File confDir,
- String siteXMLFilename)
- throws BadCommandArgumentsException, IOException {
- Configuration siteConf;
- File siteXML = new File(confDir, siteXMLFilename);
- if (!siteXML.exists()) {
- throw new BadCommandArgumentsException(
- "Configuration directory %s doesn't contain %s - listing is %s",
- confDir, siteXMLFilename, SliderUtils.listDir(confDir));
- }
-
- //now read it in
- siteConf = ConfigHelper.loadConfFromFile(siteXML);
- log.info("{} file is at {}", siteXMLFilename, siteXML);
- log.info(ConfigHelper.dumpConfigToString(siteConf));
- return siteConf;
- }
-
- /**
- * No-op implementation of this method.
- */
- @Override
- public void initializeApplicationConfiguration(
- AggregateConf instanceDefinition, SliderFileSystem fileSystem,
- String roleGroup)
- throws IOException, SliderException {
- }
-
- /**
- * No-op implementation of this method.
- *
- * {@inheritDoc}
- */
- @Override
- public void validateApplicationConfiguration(AggregateConf instance,
- File confDir,
- boolean secure)
- throws IOException, SliderException {
-
- }
-
- /**
- * Scan through the roles and see if it is supported.
- * @param role role to look for
- * @return true if the role is known about -and therefore
- * that a launcher thread can be deployed to launch it
- */
- @Override
- public boolean isSupportedRole(String role) {
- Collection<ProviderRole> roles = getRoles();
- for (ProviderRole providedRole : roles) {
- if (providedRole.name.equals(role)) {
- return true;
- }
- }
- return false;
- }
-
- /**
- * override point to allow a process to start executing in this container
- * @param instanceDefinition cluster description
- * @param confDir configuration directory
- * @param env environment
- * @param execInProgress the callback for the exec events
- * @return false
- * @throws IOException
- * @throws SliderException
- */
- @Override
- public boolean exec(AggregateConf instanceDefinition,
- File confDir,
- Map<String, String> env,
- ProviderCompleted execInProgress) throws IOException, SliderException {
- return false;
- }
-
- @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
- @Override // ExitCodeProvider
- public int getExitCode() {
- Throwable cause = getFailureCause();
- if (cause != null) {
- //failed for some reason
- if (cause instanceof ExitCodeProvider) {
- return ((ExitCodeProvider) cause).getExitCode();
- }
- }
- ForkedProcessService lastProc = latestProcess();
- if (lastProc == null || !lastProc.isProcessTerminated()) {
- return 0;
- } else {
- return lastProc.getExitCode();
- }
- }
-
- /**
- * Return the latest forked process service that ran
- * @return the forkes service
- */
- protected ForkedProcessService latestProcess() {
- Service current = getActiveService();
- Service prev = getPreviousService();
-
- Service latest = current != null ? current : prev;
- if (latest instanceof ForkedProcessService) {
- return (ForkedProcessService) latest;
- } else {
- //its a composite object, so look inside it for a process
- if (latest instanceof ServiceParent) {
- return getFPSFromParentService((ServiceParent) latest);
- } else {
- //no match
- return null;
- }
- }
- }
-
-
- /**
- * Given a parent service, find the one that is a forked process
- * @param serviceParent parent
- * @return the forked process service or null if there is none
- */
- protected ForkedProcessService getFPSFromParentService(ServiceParent serviceParent) {
- List<Service> services = serviceParent.getServices();
- for (Service s : services) {
- if (s instanceof ForkedProcessService) {
- return (ForkedProcessService) s;
- }
- }
- return null;
- }
-
- /**
- * if we are already running, start this service
- */
- protected void maybeStartCommandSequence() {
- if (isInState(STATE.STARTED)) {
- startNextService();
- }
- }
-
- /**
- * Create a new forked process service with the given
- * name, environment and command list -then add it as a child
- * for execution in the sequence.
- *
- * @param name command name
- * @param env environment
- * @param commands command line
- * @throws IOException
- * @throws SliderException
- */
- protected ForkedProcessService queueCommand(String name,
- Map<String, String> env,
- List<String> commands) throws
- IOException,
- SliderException {
- ForkedProcessService process = buildProcess(name, env, commands);
- //register the service for lifecycle management; when this service
- //is terminated, so is the master process
- addService(process);
- return process;
- }
-
- public ForkedProcessService buildProcess(String name,
- Map<String, String> env,
- List<String> commands) throws
- IOException,
- SliderException {
- ForkedProcessService process;
- process = new ForkedProcessService(name);
- process.init(getConfig());
- process.build(env, commands);
- return process;
- }
-
- /*
- * Build the provider status, can be empty
- * @return the provider status - map of entries to add to the info section
- */
- @Override
- public Map<String, String> buildProviderStatus() {
- return new HashMap<String, String>();
- }
-
- /*
- Build the monitor details. The base implementation includes all the external URL endpoints
- in the external view
- */
- @Override
- public Map<String, MonitorDetail> buildMonitorDetails(ClusterDescription clusterDesc) {
- Map<String, MonitorDetail> details = new LinkedHashMap<String, MonitorDetail>();
-
- // add in all the endpoints
- buildEndpointDetails(details);
-
- return details;
- }
-
- @Override
- public void buildEndpointDetails(Map<String, MonitorDetail> details) {
- ServiceRecord self = yarnRegistry.getSelfRegistration();
-
- List<Endpoint> externals = self.external;
- for (Endpoint endpoint : externals) {
- String addressType = endpoint.addressType;
- if (AddressTypes.ADDRESS_URI.equals(addressType)) {
- try {
- List<URL> urls = RegistryTypeUtils.retrieveAddressURLs(endpoint);
- if (!urls.isEmpty()) {
- details.put(endpoint.api, new MonitorDetail(urls.get(0).toString(), true));
- }
- } catch (InvalidRecordException | MalformedURLException ignored) {
- // Ignored
- }
-
- }
-
- }
- }
-
- @Override
- public void applyInitialRegistryDefinitions(URL amWebURI,
- ServiceRecord serviceRecord)
- throws IOException {
- this.amWebAPI = amWebURI;
- }
-
- /**
- * {@inheritDoc}
- *
- *
- * @return The base implementation returns the most recent containers first.
- */
- @Override
- public ContainerReleaseSelector createContainerReleaseSelector() {
- return new MostRecentContainerReleaseSelector();
- }
-
- @Override
- public void releaseAssignedContainer(ContainerId containerId) {
- // no-op
- }
-
- @Override
- public void addContainerRequest(AMRMClient.ContainerRequest req) {
- // no-op
- }
-
- @Override
- public void cancelSingleRequest(AMRMClient.ContainerRequest request) {
- // no-op
- }
-
- @Override
- public int cancelContainerRequests(Priority priority1,
- Priority priority2,
- int count) {
- return 0;
- }
-
- @Override
- public void updateBlacklist(List<String> blacklistAdditions,
- List<String> blacklistRemovals) {
- // no-op
- }
-
- @Override
- public void execute(List<AbstractRMOperation> operations) {
- for (AbstractRMOperation operation : operations) {
- operation.execute(this);
- }
- }
- /**
- * No-op implementation of this method.
- */
- @Override
- public void rebuildContainerDetails(List<Container> liveContainers,
- String applicationId, Map<Integer, ProviderRole> providerRoles) {
- }
-
- @Override
- public boolean processContainerStatus(ContainerId containerId,
- ContainerStatus status) {
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c242ac2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
index 9767430..b07fc29 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
@@ -19,9 +19,6 @@
package org.apache.slider.providers;
import org.apache.hadoop.conf.Configuration;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.ConfTree;
-import org.apache.slider.core.exceptions.SliderException;
import java.util.List;
public interface ProviderCore {
@@ -31,13 +28,4 @@ public interface ProviderCore {
List<ProviderRole> getRoles();
Configuration getConf();
-
- /**
- * Verify that an instance definition is considered valid by the provider
- * @param instanceDefinition instance definition
- * @throws SliderException if the configuration is not valid
- */
- void validateInstanceDefinition(AggregateConf instanceDefinition) throws
- SliderException;
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c242ac2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
index 761ac0f..e0299e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
@@ -18,7 +18,7 @@
package org.apache.slider.providers;
-import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.api.resource.Component;
/**
* Provider role and key for use in app requests.
@@ -34,16 +34,8 @@ public final class ProviderRole {
public int nodeFailureThreshold;
public final long placementTimeoutSeconds;
public final String labelExpression;
+ public final Component component;
- public ProviderRole(String name, int id) {
- this(name,
- name,
- id,
- PlacementPolicy.DEFAULT,
- ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD,
- ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS,
- ResourceKeys.DEF_YARN_LABEL_EXPRESSION);
- }
/**
* Create a provider role
@@ -67,7 +59,7 @@ public final class ProviderRole {
policy,
nodeFailureThreshold,
placementTimeoutSeconds,
- labelExpression);
+ labelExpression, null);
}
/**
@@ -87,7 +79,8 @@ public final class ProviderRole {
int policy,
int nodeFailureThreshold,
long placementTimeoutSeconds,
- String labelExpression) {
+ String labelExpression,
+ Component component) {
this.name = name;
if (group == null) {
this.group = name;
@@ -99,6 +92,8 @@ public final class ProviderRole {
this.nodeFailureThreshold = nodeFailureThreshold;
this.placementTimeoutSeconds = placementTimeoutSeconds;
this.labelExpression = labelExpression;
+ this.component = component;
+
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c242ac2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index 4ca9326..c80de7f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -18,18 +18,15 @@
package org.apache.slider.providers;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.resource.Application;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.MapOperations;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.core.main.ExitCodeProvider;
@@ -45,128 +42,17 @@ import java.net.URL;
import java.util.List;
import java.util.Map;
-public interface ProviderService extends ProviderCore,
- Service,
- RMOperationHandlerActions,
- ExitCodeProvider {
+public interface ProviderService extends Service {
/**
* Set up the entire container launch context
- * @param containerLauncher
- * @param instanceDefinition
- * @param container
- * @param providerRole
- * @param sliderFileSystem
- * @param generatedConfPath
- * @param appComponent
- * @param containerTmpDirPath
*/
void buildContainerLaunchContext(ContainerLauncher containerLauncher,
- AggregateConf instanceDefinition,
- Container container,
- ProviderRole providerRole,
- SliderFileSystem sliderFileSystem,
- Path generatedConfPath,
- MapOperations resourceComponent,
- MapOperations appComponent,
- Path containerTmpDirPath) throws
- IOException,
- SliderException;
+ Application application, Container container, ProviderRole providerRole,
+ SliderFileSystem sliderFileSystem) throws IOException, SliderException;
- /**
- * Notify the providers of container completion
- * @param containerId container that has completed
- */
- void notifyContainerCompleted(ContainerId containerId);
-
- /**
- * Execute a process in the AM
- * @param instanceDefinition cluster description
- * @param confDir configuration directory
- * @param env environment
- * @param execInProgress the callback for the exec events
- * @return true if a process was actually started
- * @throws IOException
- * @throws SliderException
- */
- boolean exec(AggregateConf instanceDefinition,
- File confDir,
- Map<String, String> env,
- ProviderCompleted execInProgress) throws IOException,
- SliderException;
-
- /**
- * Scan through the roles and see if it is supported.
- * @param role role to look for
- * @return true if the role is known about -and therefore
- * that a launcher thread can be deployed to launch it
- */
- boolean isSupportedRole(String role);
-
- /**
- * Load a specific XML configuration file for the provider config
- * @param confDir configuration directory
- * @return a configuration to be included in status
- * @throws BadCommandArgumentsException
- * @throws IOException
- */
- Configuration loadProviderConfigurationInformation(File confDir)
- throws BadCommandArgumentsException, IOException;
- /**
- * The application configuration should be initialized here
- *
- * @param instanceDefinition
- * @param fileSystem
- * @param roleGroup
- * @throws IOException
- * @throws SliderException
- */
- void initializeApplicationConfiguration(AggregateConf instanceDefinition,
- SliderFileSystem fileSystem, String roleGroup) throws IOException,
- SliderException;
-
- /**
- * This is a validation of the application configuration on the AM.
- * Here is where things like the existence of keytabs and other
- * not-seen-client-side properties can be tested, before
- * the actual process is spawned.
- * @param instanceDefinition clusterSpecification
- * @param confDir configuration directory
- * @param secure flag to indicate that secure mode checks must exist
- * @throws IOException IO problemsn
- * @throws SliderException any failure
- */
- void validateApplicationConfiguration(AggregateConf instanceDefinition,
- File confDir,
- boolean secure
- ) throws IOException, SliderException;
-
- /*
- * Build the provider status, can be empty
- * @return the provider status - map of entries to add to the info section
- */
- Map<String, String> buildProviderStatus();
-
- /**
- * Build a map of data intended for the AM webapp that is specific
- * about this provider. The key is some text to be displayed, and the
- * value can be a URL that will create an anchor over the key text.
- *
- * If no anchor is needed/desired, insert the key with a null value.
- * @return the details
- */
- Map<String, MonitorDetail> buildMonitorDetails(ClusterDescription clusterSpec);
-
- /**
- * Get a human friendly name for web UIs and messages
- * @return a name string. Default is simply the service instance name.
- */
- String getHumanName();
-
- public void bind(StateAccessForProviders stateAccessor,
- QueueAccess queueAccess,
- List<Container> liveContainers);
+ void setAMState(StateAccessForProviders stateAccessForProviders);
/**
* Bind to the YARN registry
@@ -175,39 +61,6 @@ public interface ProviderService extends ProviderCore,
void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry);
/**
- * Build up the endpoint details for this service
- * @param details
- */
- void buildEndpointDetails(Map<String, MonitorDetail> details);
-
- /**
- * Prior to going live -register the initial service registry data
- * @param amWebURI URL to the AM. This may be proxied, so use relative paths
- * @param serviceRecord service record to build up
- */
- void applyInitialRegistryDefinitions(URL amWebURI,
- ServiceRecord serviceRecord)
- throws IOException;
-
- /**
- * Create the container release selector for this provider...any policy
- * can be implemented
- * @return the selector to use for choosing containers.
- */
- ContainerReleaseSelector createContainerReleaseSelector();
-
- /**
- * On AM restart (for whatever reason) this API is required to rebuild the AM
- * internal state with the containers which were already assigned and running
- *
- * @param liveContainers
- * @param applicationId
- * @param providerRoles
- */
- void rebuildContainerDetails(List<Container> liveContainers,
- String applicationId, Map<Integer, ProviderRole> providerRoles);
-
- /**
* Process container status
* @return true if status needs to be requested again, false otherwise
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c242ac2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index f33db9b..f8ec976 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -18,7 +18,6 @@
package org.apache.slider.providers;
-import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
@@ -37,24 +36,21 @@ import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.OptionKeys;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.RoleKeys;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.Component;
+import org.apache.slider.api.resource.ConfigFile;
+import org.apache.slider.api.resource.Configuration;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.core.exceptions.NoSuchNodeException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.core.registry.docstore.ConfigFormat;
-import org.apache.slider.core.registry.docstore.ConfigUtils;
-import org.apache.slider.core.registry.docstore.ExportEntry;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
-import org.apache.slider.core.registry.docstore.PublishedExports;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
@@ -66,16 +62,10 @@ import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Date;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
import java.util.regex.Pattern;
/**
@@ -114,7 +104,7 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
*/
public static boolean addProviderJar(
Map<String, LocalResource> providerResources,
- Object provider,
+ Class providerClass,
String jarName,
SliderFileSystem sliderFileSystem,
Path tempPath,
@@ -125,7 +115,7 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
try {
SliderUtils.putJar(providerResources,
sliderFileSystem,
- provider.getClass(),
+ providerClass,
tempPath,
libdir,
jarName);
@@ -138,32 +128,6 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
}
}
}
-
- /**
- * Add/overwrite the agent tarball (overwritten every time application is
- * restarted).
- * @param provider an instance of a provider class
- * @param tarName name of the tarball to upload
- * @param sliderFileSystem the file system
- * @param agentDir directory to upload to
- * @return true the location could be determined and the file added
- * @throws IOException if the upload fails
- */
- public static boolean addAgentTar(Object provider,
- String tarName,
- SliderFileSystem sliderFileSystem,
- Path agentDir) throws
- IOException {
- File localFile = SliderUtils.findContainingJar(provider.getClass());
- if(localFile != null) {
- String parentDir = localFile.getParent();
- Path agentTarPath = new Path(parentDir, tarName);
- sliderFileSystem.getFileSystem().copyFromLocalFile(false, true,
- agentTarPath, agentDir);
- return true;
- }
- return false;
- }
/**
* Loads all dependency jars from the default path.
@@ -193,132 +157,24 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
libDir, libLocalSrcDir);
}
-
- /**
- * Validate the requested number of instances of a component.
- * <p>
- * If max <= 0: min <= count
- * If max > 0: min <= count <= max
- * @param instanceDescription configuration
- * @param name node class name
- * @param min requested heap size
- * @param max maximum value.
- * @throws BadCommandArgumentsException if the values are out of range
- */
- public void validateNodeCount(AggregateConf instanceDescription,
- String name, int min, int max)
- throws BadCommandArgumentsException {
- MapOperations component =
- instanceDescription.getResourceOperations().getComponent(name);
- int count;
- if (component == null) {
- count = 0;
- } else {
- count = component.getOptionInt(ResourceKeys.COMPONENT_INSTANCES, 0);
- }
- validateNodeCount(name, count, min, max);
- }
-
- /**
- * Validate the count is between min and max.
- * <p>
- * If max <= 0: min <= count
- * If max > 0: min <= count <= max
- * @param name node class name
- * @param count requested node count
- * @param min requested heap size
- * @param max maximum value.
- * @throws BadCommandArgumentsException if the values are out of range
- */
- public void validateNodeCount(String name,
- int count,
- int min,
- int max) throws BadCommandArgumentsException {
- if (count < min) {
- throw new BadCommandArgumentsException(
- "requested no of %s nodes: %d is below the minimum of %d", name, count,
- min);
- }
- if (max > 0 && count > max) {
- throw new BadCommandArgumentsException(
- "requested no of %s nodes: %d is above the maximum of %d", name, count,
- max);
- }
- }
-
- /**
- * Copy options beginning with "site.configName." prefix from options map
- * to sitexml map, removing the prefix and substituting the tokens
- * specified in the tokenMap.
- * @param options source map
- * @param sitexml destination map
- * @param configName optional ".configName" portion of the prefix
- * @param tokenMap key/value pairs to substitute into the option values
- */
- public void propagateSiteOptions(Map<String, String> options,
- Map<String, String> sitexml,
- String configName,
- Map<String,String> tokenMap) {
- String prefix = OptionKeys.SITE_XML_PREFIX +
- (!configName.isEmpty() ? configName + "." : "");
- propagateOptions(options, sitexml, tokenMap, prefix);
- }
-
- /**
- * Copy options beginning with prefix from options map
- * to sitexml map, removing the prefix and substituting the tokens
- * specified in the tokenMap.
- * @param options source map
- * @param sitexml destination map
- * @param tokenMap key/value pairs to substitute into the option values
- * @param prefix which options to copy to destination map
- */
- public void propagateOptions(Map<String, String> options,
- Map<String, String> sitexml,
- Map<String,String> tokenMap,
- String prefix) {
- for (Map.Entry<String, String> entry : options.entrySet()) {
- String key = entry.getKey();
- if (key.startsWith(prefix)) {
- String envName = key.substring(prefix.length());
- if (!envName.isEmpty()) {
- String value = entry.getValue();
- if (tokenMap != null) {
- for (Map.Entry<String,String> token : tokenMap.entrySet()) {
- value = value.replaceAll(Pattern.quote(token.getKey()),
- token.getValue());
- }
- }
- sitexml.put(envName, value);
- }
- }
- }
- }
-
- /**
- * Substitute tokens into option map values, returning a new map.
- * @param options source map
- * @param tokenMap key/value pairs to substitute into the option values
- * @return map with substituted values
- */
- public Map<String, String> filterSiteOptions(Map<String, String> options,
+ // Build key -> value map
+ // value will be substituted by corresponding data in tokenMap
+ public Map<String, String> substituteConfigs(Map<String, String> configs,
Map<String, String> tokenMap) {
- String prefix = OptionKeys.SITE_XML_PREFIX;
String format = "${%s}";
Map<String, String> filteredOptions = new HashMap<>();
- for (Map.Entry<String, String> entry : options.entrySet()) {
+ for (Map.Entry<String, String> entry : configs.entrySet()) {
String key = entry.getKey();
- if (key.startsWith(prefix)) {
- String value = entry.getValue();
- if (tokenMap != null) {
- for (Map.Entry<String,String> token : tokenMap.entrySet()) {
- value = value.replaceAll(Pattern.quote(token.getKey()),
- token.getValue());
- }
+ String value = entry.getValue();
+ if (tokenMap != null) {
+ for (Map.Entry<String, String> token : tokenMap.entrySet()) {
+ value =
+ value.replaceAll(Pattern.quote(token.getKey()), token.getValue());
}
- filteredOptions.put(String.format(format, key), value);
}
+ filteredOptions.put(String.format(format, key), value);
}
+
return filteredOptions;
}
@@ -345,28 +201,27 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
return intVal;
}
+
/**
* Localize the service keytabs for the application.
* @param launcher container launcher
- * @param instanceDefinition app specification
* @param fileSystem file system
- * @param clusterName app name
* @throws IOException trouble uploading to HDFS
*/
public void localizeServiceKeytabs(ContainerLauncher launcher,
- AggregateConf instanceDefinition, SliderFileSystem fileSystem,
- String clusterName) throws IOException {
- ConfTreeOperations appConf = instanceDefinition.getAppConfOperations();
- String keytabPathOnHost = appConf.getComponent(COMPONENT_AM).get(
- SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
+ SliderFileSystem fileSystem, Application application) throws IOException {
+
+ Configuration conf = application.getConfiguration();
+ String keytabPathOnHost =
+ conf.getProperty(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
if (SliderUtils.isUnset(keytabPathOnHost)) {
- String amKeytabName = appConf.getComponent(COMPONENT_AM).get(
- SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
- String keytabDir = appConf.getComponent(COMPONENT_AM).get(
- SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR);
+ String amKeytabName =
+ conf.getProperty(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
+ String keytabDir =
+ conf.getProperty(SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR);
// we need to localize the keytab files in the directory
Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null,
- clusterName);
+ application.getName());
boolean serviceKeytabsDeployed = false;
if (fileSystem.getFileSystem().exists(keytabDirPath)) {
FileStatus[] keytabs = fileSystem.getFileSystem().listStatus(
@@ -395,591 +250,119 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
}
- /**
- * Upload a local file to the cluster security dir in HDFS. If the file
- * already exists, it is not replaced.
- * @param resource file to upload
- * @param fileSystem file system
- * @param clusterName app name
- * @return Path of the uploaded file
- * @throws IOException file cannot be uploaded
- */
- private Path uploadSecurityResource(File resource,
- SliderFileSystem fileSystem, String clusterName) throws IOException {
- Path certsDir = fileSystem.buildClusterSecurityDirPath(clusterName);
- return uploadResource(resource, fileSystem, certsDir);
- }
-
- /**
- * Upload a local file to the cluster resources dir in HDFS. If the file
- * already exists, it is not replaced.
- * @param resource file to upload
- * @param fileSystem file system
- * @param roleName optional subdirectory (for component-specific resources)
- * @param clusterName app name
- * @return Path of the uploaded file
- * @throws IOException file cannot be uploaded
- */
- private Path uploadResource(File resource, SliderFileSystem fileSystem,
- String roleName, String clusterName) throws IOException {
- Path dir;
- if (roleName == null) {
- dir = fileSystem.buildClusterResourcePath(clusterName);
- } else {
- dir = fileSystem.buildClusterResourcePath(clusterName, roleName);
- }
- return uploadResource(resource, fileSystem, dir);
- }
-
- /**
- * Upload a local file to a specified HDFS directory. If the file already
- * exists, it is not replaced.
- * @param resource file to upload
- * @param fileSystem file system
- * @param parentDir destination directory in HDFS
- * @return Path of the uploaded file
- * @throws IOException file cannot be uploaded
- */
- private synchronized Path uploadResource(File resource,
- SliderFileSystem fileSystem, Path parentDir) throws IOException {
- if (!fileSystem.getFileSystem().exists(parentDir)) {
- fileSystem.getFileSystem().mkdirs(parentDir,
+ // 1. Create all config files for a component on hdfs for localization
+ // 2. Add the config file to localResource
+ //TODO handle Template format config file
+ public void createConfigFileAndAddLocalResource(ContainerLauncher launcher,
+ SliderFileSystem fs, Component component,
+ Map<String, String> tokensForSubstitution,
+ StateAccessForProviders amState) throws IOException {
+ Path compDir =
+ new Path(new Path(fs.getAppDir(), "components"), component.getName());
+ if (!fs.getFileSystem().exists(compDir)) {
+ fs.getFileSystem().mkdirs(compDir,
new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
- }
- Path destPath = new Path(parentDir, resource.getName());
- if (!fileSystem.getFileSystem().exists(destPath)) {
- FSDataOutputStream os = null;
- try {
- os = fileSystem.getFileSystem().create(destPath);
- byte[] contents = FileUtils.readFileToByteArray(resource);
- os.write(contents, 0, contents.length);
- os.flush();
- } finally {
- IOUtils.closeStream(os);
- }
- log.info("Uploaded {} to localization path {}", resource, destPath);
+ log.info("Creating component dir: " + compDir);
} else {
- log.info("Resource {} already existed at localization path {}", resource,
- destPath);
- }
-
- while (!fileSystem.getFileSystem().exists(destPath)) {
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- // ignore
- }
- }
-
- fileSystem.getFileSystem().setPermission(destPath,
- new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE));
-
- return destPath;
- }
-
- /**
- * Write a configuration property map to a local file in a specified format.
- * @param fileSystem file system
- * @param file destination file
- * @param configFormat file format
- * @param configFileDN file description
- * @param config properties to save to the file
- * @param clusterName app name
- * @throws IOException file cannot be created
- */
- private synchronized void createConfigFile(SliderFileSystem fileSystem,
- File file, ConfigFormat configFormat, String configFileDN,
- Map<String, String> config, String clusterName) throws IOException {
- if (file.exists()) {
- log.info("Skipping writing {} file {} because it already exists",
- configFormat, file);
- return;
- }
- log.info("Writing {} file {}", configFormat, file);
-
- ConfigUtils.prepConfigForTemplateOutputter(configFormat, config,
- fileSystem, clusterName, file.getName());
- PublishedConfiguration publishedConfiguration =
- new PublishedConfiguration(configFileDN,
- config.entrySet());
- PublishedConfigurationOutputter configurationOutputter =
- PublishedConfigurationOutputter.createOutputter(configFormat,
- publishedConfiguration);
- configurationOutputter.save(file);
- }
-
- /**
- * Determine config files requested in the appConf, generate the files, and
- * localize them.
- * @param launcher container launcher
- * @param roleName component name
- * @param roleGroup component group
- * @param appConf app configurations
- * @param configs configurations grouped by config name
- * @param env environment variables
- * @param fileSystem file system
- * @param clusterName app name
- * @throws IOException file(s) cannot be uploaded
- * @throws BadConfigException file name not specified or file format not
- * supported
- */
- public void localizeConfigFiles(ContainerLauncher launcher,
- String roleName, String roleGroup,
- ConfTreeOperations appConf,
- Map<String, Map<String, String>> configs,
- MapOperations env,
- SliderFileSystem fileSystem,
- String clusterName)
- throws IOException, BadConfigException {
- for (Entry<String, Map<String, String>> configEntry : configs.entrySet()) {
- String configFileName = appConf.getComponentOpt(roleGroup,
- OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys
- .NAME_SUFFIX, null);
- String configFileType = appConf.getComponentOpt(roleGroup,
- OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys
- .TYPE_SUFFIX, null);
- if (configFileName == null && configFileType == null) {
- // config file not requested, so continue
- continue;
- }
- if (configFileName == null) {
- throw new BadConfigException("Config file name null for " +
- configEntry.getKey());
- }
- if (configFileType == null) {
- throw new BadConfigException("Config file type null for " +
- configEntry.getKey());
- }
- ConfigFormat configFormat = ConfigFormat.resolve(configFileType);
- if (configFormat == null) {
- throw new BadConfigException("Config format " + configFileType +
- " doesn't exist");
- }
- boolean perComponent = appConf.getComponentOptBool(roleGroup,
- OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys
- .PER_COMPONENT, false);
- boolean perGroup = appConf.getComponentOptBool(roleGroup,
- OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys
- .PER_GROUP, false);
-
- localizeConfigFile(launcher, roleName, roleGroup, configEntry.getKey(),
- configFormat, configFileName, configs, env, fileSystem,
- clusterName, perComponent, perGroup);
- }
- }
-
- /**
- * Create and localize a config file.
- * @param launcher container launcher
- * @param roleName component name
- * @param roleGroup component group
- * @param configFileDN config description/name
- * @param configFormat config format
- * @param configFileName config file name
- * @param configs configs grouped by config description/name
- * @param env environment variables
- * @param fileSystem file system
- * @param clusterName app name
- * @param perComponent true if file should be created per unique component
- * @param perGroup true if file should be created per component group
- * @throws IOException file cannot be uploaded
- */
- public void localizeConfigFile(ContainerLauncher launcher,
- String roleName, String roleGroup,
- String configFileDN, ConfigFormat configFormat, String configFileName,
- Map<String, Map<String, String>> configs,
- MapOperations env,
- SliderFileSystem fileSystem,
- String clusterName,
- boolean perComponent,
- boolean perGroup)
- throws IOException {
- if (launcher == null) {
+ log.info("Component conf dir already exists: " + compDir);
return;
}
- Map<String, String> config = ConfigUtils.replacePropsInConfig(
- configs.get(configFileDN), env.options);
- String fileName = ConfigUtils.replaceProps(config, configFileName);
- File localFile = new File(RESOURCE_DIR);
- if (!localFile.exists()) {
- if (!localFile.mkdir() && !localFile.exists()) {
- throw new IOException(RESOURCE_DIR + " could not be created!");
- }
- }
-
- String folder = null;
- if (perComponent) {
- folder = roleName;
- } else if (perGroup) {
- folder = roleGroup;
- }
- if (folder != null) {
- localFile = new File(localFile, folder);
- if (!localFile.exists()) {
- if (!localFile.mkdir() && !localFile.exists()) {
- throw new IOException(localFile + " could not be created!");
- }
- }
- }
- localFile = new File(localFile, new File(fileName).getName());
-
- log.info("Localizing {} configs to config file {} (destination {}) " +
- "based on {} configs", config.size(), localFile, fileName,
- configFileDN);
- if (!localFile.exists()) {
- createConfigFile(fileSystem, localFile, configFormat, configFileDN,
- config, clusterName);
- } else {
- log.info("Local {} file {} already exists", configFormat, localFile);
- }
- Path destPath = uploadResource(localFile, fileSystem, folder, clusterName);
- LocalResource configResource = fileSystem.createAmResource(destPath,
- LocalResourceType.FILE);
-
- File destFile = new File(fileName);
- if (destFile.isAbsolute()) {
- launcher.addLocalResource(
- RESOURCE_DIR + "/" + destFile.getName(),
- configResource, fileName);
- } else {
- launcher.addLocalResource(APP_CONF_DIR + "/" + fileName,
- configResource);
- }
- }
- /**
- * Localize application tarballs and other resources requested by the app.
- * @param launcher container launcher
- * @param fileSystem file system
- * @param appConf app configurations
- * @param roleGroup component group
- * @param clusterName app name
- * @throws IOException resources cannot be uploaded
- * @throws BadConfigException package name or type is not specified
- */
- public void localizePackages(ContainerLauncher launcher,
- SliderFileSystem fileSystem, ConfTreeOperations appConf, String roleGroup,
- String clusterName) throws IOException, BadConfigException {
- for (Entry<String, Map<String, String>> pkg :
- getPackages(roleGroup, appConf).entrySet()) {
- String pkgName = pkg.getValue().get(OptionKeys.NAME_SUFFIX);
- String pkgType = pkg.getValue().get(OptionKeys.TYPE_SUFFIX);
- Path pkgPath = fileSystem.buildResourcePath(pkgName);
- if (!fileSystem.isFile(pkgPath)) {
- pkgPath = fileSystem.buildResourcePath(clusterName,
- pkgName);
- }
- if (!fileSystem.isFile(pkgPath)) {
- throw new IOException("Package doesn't exist as a resource: " +
- pkgName);
+ for (ConfigFile configFile : component.getConfiguration().getFiles()) {
+ String fileName = configFile.getSrcFile();
+ // substitute file name
+ for (Map.Entry<String, String> token : tokensForSubstitution.entrySet()) {
+ configFile.setDestFile(configFile.getDestFile()
+ .replaceAll(Pattern.quote(token.getKey()), token.getValue()));
}
- log.info("Adding resource {}", pkgName);
- LocalResourceType type = LocalResourceType.FILE;
- if ("archive".equals(pkgType)) {
- type = LocalResourceType.ARCHIVE;
- }
- LocalResource packageResource = fileSystem.createAmResource(
- pkgPath, type);
- launcher.addLocalResource(APP_PACKAGES_DIR, packageResource);
- }
- }
-
- /**
- * Build a map of configuration description/name to configuration key/value
- * properties, with all known tokens substituted into the property values.
- * @param appConf app configurations
- * @param internalsConf internal configurations
- * @param containerId container ID
- * @param roleName component name
- * @param roleGroup component group
- * @param amState access to AM state
- * @return configuration properties grouped by config description/name
- */
- public Map<String, Map<String, String>> buildConfigurations(
- ConfTreeOperations appConf, ConfTreeOperations internalsConf,
- String containerId, String clusterName, String roleName, String roleGroup,
- StateAccessForProviders amState) {
-
- Map<String, Map<String, String>> configurations = new TreeMap<>();
- Map<String, String> tokens = getStandardTokenMap(appConf,
- internalsConf, roleName, roleGroup, containerId, clusterName);
-
- Set<String> configs = new HashSet<>();
- configs.addAll(getApplicationConfigurationTypes(roleGroup, appConf));
- configs.addAll(getSystemConfigurationsRequested(appConf));
-
- for (String configType : configs) {
- addNamedConfiguration(configType, appConf.getGlobalOptions().options,
- configurations, tokens, amState);
- if (appConf.getComponent(roleGroup) != null) {
- addNamedConfiguration(configType,
- appConf.getComponent(roleGroup).options, configurations, tokens,
- amState);
- }
- }
-
- //do a final replacement of re-used configs
- dereferenceAllConfigs(configurations);
-
- return configurations;
- }
-
- /**
- * Substitute "site." prefixed configuration values into other configuration
- * values where needed. The format for these substitutions is that
- * {@literal ${@//site/configDN/key}} will be replaced by the value for the
- * "site.configDN.key" property.
- * @param configurations configuration properties grouped by config
- * description/name
- */
- public void dereferenceAllConfigs(
- Map<String, Map<String, String>> configurations) {
- Map<String, String> allConfigs = new HashMap<>();
- String lookupFormat = "${@//site/%s/%s}";
- for (Map.Entry<String, Map<String, String>> entry : configurations.entrySet()) {
- Map<String, String> configBucket = entry.getValue();
- for(Map.Entry<String, String> config: configBucket.entrySet()) {
- allConfigs.put(String.format(lookupFormat, entry.getKey(), config.getKey()),
- config.getValue());
- }
- }
-
- boolean finished = false;
- while (!finished) {
- finished = true;
- for (Map.Entry<String, String> entry : allConfigs.entrySet()) {
- String configValue = entry.getValue();
- for (Map.Entry<String, String> lookUpEntry : allConfigs.entrySet()) {
- String lookUpValue = lookUpEntry.getValue();
- if (lookUpValue.contains("${@//site/")) {
- continue;
- }
- String lookUpKey = lookUpEntry.getKey();
- if (configValue != null && configValue.contains(lookUpKey)) {
- configValue = configValue.replace(lookUpKey, lookUpValue);
- }
- }
- if (configValue != null && !configValue.equals(entry.getValue())) {
- finished = false;
- allConfigs.put(entry.getKey(), configValue);
- }
- }
- }
- for (Map.Entry<String, Map<String, String>> configEntry : configurations
- .entrySet()) {
- Map<String, String> configBucket = configEntry.getValue();
- for (Map.Entry<String, String> entry: configBucket.entrySet()) {
- String configName = entry.getKey();
- String configValue = entry.getValue();
- for (Map.Entry<String, String> lookUpEntry : allConfigs.entrySet()) {
- String lookUpValue = lookUpEntry.getValue();
- if (lookUpValue.contains("${@//site/")) {
- continue;
- }
- String lookUpKey = lookUpEntry.getKey();
- if (configValue != null && configValue.contains(lookUpKey)) {
- configValue = configValue.replace(lookUpKey, lookUpValue);
+ // substitute configs
+ substituteConfigs(configFile.getProps(), tokensForSubstitution);
+
+ // write configs onto hdfs
+ PublishedConfiguration publishedConfiguration =
+ new PublishedConfiguration(fileName,
+ configFile.getProps().entrySet());
+ Path remoteFile = new Path(compDir, fileName);
+ if (!fs.getFileSystem().exists(remoteFile)) {
+ synchronized (this) {
+ if (!fs.getFileSystem().exists(remoteFile)) {
+ PublishedConfigurationOutputter configurationOutputter =
+ PublishedConfigurationOutputter.createOutputter(
+ ConfigFormat.resolve(configFile.getType().toString()),
+ publishedConfiguration);
+ FSDataOutputStream os = null;
+ try {
+ os = fs.getFileSystem().create(remoteFile);
+ configurationOutputter.save(os);
+ os.flush();
+ log.info("Created config file on hdfs: " + remoteFile);
+ } finally {
+ IOUtils.closeStream(os);
+ }
}
}
- configBucket.put(configName, configValue);
- }
- }
- }
-
- /**
- * Return a set of configuration description/names represented in the app.
- * configuration
- * @param roleGroup component group
- * @param appConf app configurations
- * @return set of configuration description/names
- */
- public Set<String> getApplicationConfigurationTypes(String roleGroup,
- ConfTreeOperations appConf) {
- Set<String> configList = new HashSet<>();
-
- String prefix = OptionKeys.CONF_FILE_PREFIX;
- String suffix = OptionKeys.TYPE_SUFFIX;
- MapOperations component = appConf.getComponent(roleGroup);
- if (component != null) {
- addConfsToList(component, configList, prefix, suffix);
- }
- addConfsToList(appConf.getGlobalOptions(), configList, prefix, suffix);
-
- return configList;
- }
-
- /**
- * Finds all configuration description/names of the form
- * prefixconfigDNsuffix in the configuration (e.g. conf.configDN.type).
- * @param confMap configuration properties
- * @param confList set containing configuration description/names
- * @param prefix configuration key prefix to match
- * @param suffix configuration key suffix to match
- */
- private void addConfsToList(Map<String, String> confMap,
- Set<String> confList, String prefix, String suffix) {
- for (Entry<String, String> entry : confMap.entrySet()) {
- String key = entry.getKey();
- if (key.startsWith(prefix) && key.endsWith(suffix)) {
- String confName = key.substring(prefix.length(),
- key.length() - suffix.length());
- if (!confName.isEmpty()) {
- confList.add(confName);
- }
- }
- }
- }
-
- /**
- * Build a map of package description/name to package key/value properties
- * (there should be two properties, type and name).
- * @param roleGroup component group
- * @param appConf app configurations
- * @return map of package description/name to package key/value properties
- * @throws BadConfigException package name or type is not specified
- */
- public Map<String, Map<String, String>> getPackages(String roleGroup,
- ConfTreeOperations appConf) throws BadConfigException {
- Map<String, Map<String, String>> packages = new HashMap<>();
- String prefix = OptionKeys.PKG_FILE_PREFIX;
- String typeSuffix = OptionKeys.TYPE_SUFFIX;
- String nameSuffix = OptionKeys.NAME_SUFFIX;
- MapOperations component = appConf.getComponent(roleGroup);
- if (component == null) {
- component = appConf.getGlobalOptions();
- }
- for (Map.Entry<String, String> entry : component.entrySet()) {
- String key = entry.getKey();
- if (key.startsWith(prefix)) {
- String confName;
- String type;
- if (key.endsWith(typeSuffix)) {
- confName = key.substring(prefix.length(), key.length() - typeSuffix.length());
- type = typeSuffix;
- } else if (key.endsWith(nameSuffix)) {
- confName = key.substring(prefix.length(), key.length() - nameSuffix.length());
- type = nameSuffix;
- } else {
- continue;
- }
- if (!packages.containsKey(confName)) {
- packages.put(confName, new HashMap<String, String>());
- }
- packages.get(confName).put(type, entry.getValue());
- }
- }
-
- for (Entry<String, Map<String, String>> pkg : packages.entrySet()) {
- if (!pkg.getValue().containsKey(OptionKeys.TYPE_SUFFIX)) {
- throw new BadConfigException("Package " + pkg.getKey() + " doesn't " +
- "have a package type");
}
- if (!pkg.getValue().containsKey(OptionKeys.NAME_SUFFIX)) {
- throw new BadConfigException("Package " + pkg.getKey() + " doesn't " +
- "have a package name");
- }
- }
-
- return packages;
- }
-
- /**
- * Return system configurations requested by the app.
- * @param appConf app configurations
- * @return set of system configurations
- */
- public Set<String> getSystemConfigurationsRequested(
- ConfTreeOperations appConf) {
- Set<String> configList = new HashSet<>();
- String configTypes = appConf.get(SYSTEM_CONFIGS);
- if (configTypes != null && configTypes.length() > 0) {
- String[] configs = configTypes.split(",");
- for (String config : configs) {
- configList.add(config.trim());
+ // Publish configs
+ amState.getPublishedSliderConfigurations()
+ .put(configFile.getSrcFile(), publishedConfiguration);
+
+ // Add resource for localization
+ LocalResource configResource =
+ fs.createAmResource(remoteFile, LocalResourceType.FILE);
+ File destFile = new File(configFile.getDestFile());
+ //TODO why to we need to differetiate RESOURCE_DIR vs APP_CONF_DIR
+ if (destFile.isAbsolute()) {
+ String symlink = RESOURCE_DIR + "/" + fileName;
+ launcher.addLocalResource(symlink, configResource,
+ configFile.getDestFile());
+ log.info("Add config file for localization: " + symlink + " -> "
+ + configResource.getResource().getFile() + ", dest mount path: "
+ + configFile.getDestFile());
+ } else {
+ String symlink = APP_CONF_DIR + "/" + fileName;
+ launcher.addLocalResource(symlink, configResource);
+ log.info("Add config file for localization: " + symlink + " -> "
+ + configResource.getResource().getFile());
}
}
-
- return configList;
- }
-
- /**
- * For a given config description/name, pull out its site configs from the
- * source config map, remove the site.configDN. prefix from them, and place
- * them into a new config map using the {@link #propagateSiteOptions} method
- * (with tokens substituted). This new k/v map is put as the value for the
- * configDN key in the configurations map.
- * @param configName config description/name
- * @param sourceConfig config containing site.* properties
- * @param configurations configuration map to be populated
- * @param tokens initial substitution tokens
- * @param amState access to AM state
- */
- private void addNamedConfiguration(String configName,
- Map<String, String> sourceConfig,
- Map<String, Map<String, String>> configurations,
- Map<String, String> tokens, StateAccessForProviders amState) {
- Map<String, String> config = new HashMap<>();
- if (configName.equals(GLOBAL_CONFIG_TAG)) {
- addDefaultGlobalConfig(config);
- }
- // add role hosts to tokens
- addRoleRelatedTokens(tokens, amState);
- propagateSiteOptions(sourceConfig, config, configName, tokens);
-
- configurations.put(configName, config);
}
/**
* Get initial token map to be substituted into config values.
* @param appConf app configurations
- * @param internals internal configurations
- * @param componentName component name
- * @param componentGroup component group
- * @param clusterName app name
- * @return tokens to replace
- */
- public Map<String, String> getStandardTokenMap(ConfTreeOperations appConf,
- ConfTreeOperations internals, String componentName,
- String componentGroup, String clusterName) {
- return getStandardTokenMap(appConf, internals, componentName,
- componentGroup, null, clusterName);
- }
-
- /**
- * Get initial token map to be substituted into config values.
- * @param appConf app configurations
- * @param internals internal configurations
* @param componentName component name
* @param componentGroup component group
* @param containerId container ID
* @param clusterName app name
* @return tokens to replace
*/
- public Map<String, String> getStandardTokenMap(ConfTreeOperations appConf,
- ConfTreeOperations internals, String componentName,
+ public Map<String, String> getStandardTokenMap(
+ Configuration appConf, Configuration componentConf, String componentName,
String componentGroup, String containerId, String clusterName) {
Map<String, String> tokens = new HashMap<>();
if (containerId != null) {
tokens.put("${CONTAINER_ID}", containerId);
}
- String nnuri = appConf.get("site.fs.defaultFS");
- tokens.put("${NN_URI}", nnuri);
- tokens.put("${NN_HOST}", URI.create(nnuri).getHost());
- tokens.put("${ZK_HOST}", appConf.get(OptionKeys.ZOOKEEPER_HOSTS));
- tokens.put("${DEFAULT_ZK_PATH}", appConf.get(OptionKeys.ZOOKEEPER_PATH));
- String prefix = appConf.getComponentOpt(componentGroup, ROLE_PREFIX,
- null);
+ String nnuri = appConf.getProperty("fs.defaultFS");
+ if (nnuri != null && !nnuri.isEmpty()) {
+ tokens.put("${NN_URI}", nnuri);
+ tokens.put("${NN_HOST}", URI.create(nnuri).getHost());
+ }
+ tokens.put("${ZK_HOST}", appConf.getProperty(OptionKeys.ZOOKEEPER_HOSTS));
+ tokens.put("${DEFAULT_ZK_PATH}", appConf.getProperty(OptionKeys.ZOOKEEPER_PATH));
+ String prefix = componentConf.getProperty(ROLE_PREFIX);
String dataDirSuffix = "";
if (prefix == null) {
prefix = "";
} else {
dataDirSuffix = "_" + SliderUtils.trimPrefix(prefix);
}
- tokens.put("${DEFAULT_DATA_DIR}", internals.getGlobalOptions()
- .getOption(InternalKeys.INTERNAL_DATA_DIR_PATH, null) + dataDirSuffix);
- tokens.put("${JAVA_HOME}", appConf.get(JAVA_HOME));
+ tokens.put("${DEFAULT_DATA_DIR}",
+ appConf.getProperty(InternalKeys.INTERNAL_DATA_DIR_PATH)
+ + dataDirSuffix);
+ tokens.put("${JAVA_HOME}", appConf.getProperty(JAVA_HOME));
tokens.put("${COMPONENT_NAME}", componentName);
tokens.put("${COMPONENT_NAME.lc}", componentName.toLowerCase());
tokens.put("${COMPONENT_PREFIX}", prefix);
@@ -1005,7 +388,7 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
* @param tokens existing tokens
* @param amState access to AM state
*/
- public void addRoleRelatedTokens(Map<String, String> tokens,
+ public void addRoleHostTokens(Map<String, String> tokens,
StateAccessForProviders amState) {
if (amState == null) {
return;
@@ -1020,26 +403,6 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
}
/**
- * Add global configuration properties.
- * @param config map where default global properties will be added
- */
- private void addDefaultGlobalConfig(Map<String, String> config) {
- config.put("app_log_dir", "${LOG_DIR}");
- config.put("app_pid_dir", "${WORK_DIR}/app/run");
- config.put("app_install_dir", "${WORK_DIR}/app/install");
- config.put("app_conf_dir", "${WORK_DIR}/" + APP_CONF_DIR);
- config.put("app_input_conf_dir", "${WORK_DIR}/" + PROPAGATED_CONF_DIR_NAME);
-
- // add optional parameters only if they are not already provided
- if (!config.containsKey("pid_file")) {
- config.put("pid_file", "${WORK_DIR}/app/run/component.pid");
- }
- if (!config.containsKey("app_root")) {
- config.put("app_root", "${WORK_DIR}/app/install");
- }
- }
-
- /**
* Return a list of hosts based on current ClusterNodes.
* @param values cluster nodes
* @param hostOnly whether host or host/server name will be added to list
@@ -1101,82 +464,4 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
containerId, e);
}
}
-
- /**
- * Publish a named property bag that may contain name-value pairs for app
- * configurations such as hbase-site.
- * @param name config file identifying name
- * @param description config file description
- * @param entries config file properties
- * @param amState access to AM state
- */
- public void publishApplicationInstanceData(String name, String description,
- Iterable<Map.Entry<String, String>> entries,
- StateAccessForProviders amState) {
- PublishedConfiguration pubconf = new PublishedConfiguration(description,
- entries);
- log.info("publishing {}", pubconf);
- amState.getPublishedSliderConfigurations().put(name, pubconf);
- }
-
- /**
- * Publish an export group.
- * @param exportGroup export groups
- * @param amState access to AM state
- * @param groupName export group name
- */
- public void publishExportGroup(
- Map<String, Set<ExportEntry>> exportGroup,
- StateAccessForProviders amState, String groupName) {
- // Publish in old format for the time being
- Map<String, String> simpleEntries = new HashMap<>();
- for (Entry<String, Set<ExportEntry>> entry : exportGroup.entrySet()) {
- Set<ExportEntry> exports = entry.getValue();
- if (SliderUtils.isNotEmpty(exports)) {
- Set<String> values = new TreeSet<>();
- for (ExportEntry export : exports) {
- values.add(export.getValue());
- }
- simpleEntries.put(entry.getKey(), StringUtils.join(",", values));
- }
- }
- publishApplicationInstanceData(groupName, groupName,
- simpleEntries.entrySet(), amState);
-
- PublishedExports exports = new PublishedExports(groupName);
- exports.setUpdated(new Date().getTime());
- exports.putValues(exportGroup.entrySet());
- amState.getPublishedExportsSet().put(groupName, exports);
- }
-
- public Map<String, String> getExports(ConfTreeOperations appConf,
- String roleGroup) {
- Map<String, String> exports = new HashMap<>();
- propagateOptions(appConf.getComponent(roleGroup).options, exports,
- null, OptionKeys.EXPORT_PREFIX);
- return exports;
- }
-
- public String getGroupKey(String roleGroup, ConfTreeOperations appConf) {
- String rolePrefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, "");
- return getNameOrGroupKey(rolePrefix, roleGroup);
- }
-
- public String getNameKey(String roleName, String roleGroup,
- ConfTreeOperations appConf) {
- String rolePrefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, "");
- return getNameOrGroupKey(rolePrefix, roleName);
- }
-
- public String getNameOrGroupKey(String rolePrefix, String roleNameOrGroup) {
- if (!rolePrefix.isEmpty()) {
- if (!roleNameOrGroup.startsWith(rolePrefix)) {
- log.warn("Something went wrong, {} doesn't start with {}",
- roleNameOrGroup, rolePrefix);
- return null;
- }
- roleNameOrGroup = roleNameOrGroup.substring(rolePrefix.length());
- }
- return roleNameOrGroup.toUpperCase(Locale.ENGLISH);
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org