You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/06/21 18:33:54 UTC
[32/50] [abbrv] hadoop git commit: YARN-6400. Remove some unneeded
code after YARN-6255. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/RestClientFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/RestClientFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/RestClientFactory.java
deleted file mode 100644
index 4286596..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/RestClientFactory.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.client.rest;
-
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.WebResource;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.slider.client.ClientRegistryBinder;
-import org.apache.slider.api.SliderApplicationApi;
-import org.apache.slider.core.registry.info.CustomRegistryConstants;
-
-import java.io.IOException;
-
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_APPLICATION;
-
-/**
- * Factory for the Rest client; hides the lookup and instantiation.
- * <p>
- *
- */
-public class RestClientFactory {
-
- private final ClientRegistryBinder binder;
- private final Client jerseyClient;
- private final String user, serviceclass, instance;
-
- public RestClientFactory(RegistryOperations operations,
- Client jerseyClient,
- String user,
- String serviceclass,
- String instance) {
- this.jerseyClient = jerseyClient;
- this.user = user;
- this.serviceclass = serviceclass;
- this.instance = instance;
- binder = new ClientRegistryBinder(operations);
- }
-
- /**
- * Locate the AM
- * @return a resource to the AM
- * @throws IOException any failure to resolve to the AM
- */
- private WebResource locateAppmaster() throws IOException {
- String restAPI = binder.lookupExternalRestAPI(user, serviceclass, instance,
- CustomRegistryConstants.AM_REST_BASE);
- return jerseyClient.resource(restAPI);
- }
-
- /**
- * Locate the slider AM then instantiate a client instance against
- * its Application API.
- * @return the instance
- * @throws IOException on any failure
- */
- public SliderApplicationApi createSliderAppApiClient() throws IOException {
- WebResource appmaster = locateAppmaster();
- return createSliderAppApiClient(appmaster);
- }
-
- /**
- * Create a Slider application API client instance against
- * its Application API.
- * @param appmaster The AM to work against.
- * @return the instance
- * @throws IOException on any failure
- */
- public SliderApplicationApi createSliderAppApiClient(WebResource appmaster) {
- WebResource appResource = appmaster.path(SLIDER_PATH_APPLICATION);
- return new SliderApplicationApiRestClient(jerseyClient, appResource);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
deleted file mode 100644
index 4c376e0..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
+++ /dev/null
@@ -1,289 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.client.rest;
-
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.ClientHandlerException;
-import com.sun.jersey.api.client.GenericType;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.representation.Form;
-import org.apache.commons.lang.StringUtils;
-import org.apache.slider.api.types.ApplicationLivenessInformation;
-import org.apache.slider.api.types.ComponentInformation;
-import org.apache.slider.api.types.ContainerInformation;
-import org.apache.slider.api.SliderApplicationApi;
-import org.apache.slider.api.types.NodeInformation;
-import org.apache.slider.api.types.NodeInformationList;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.ConfTree;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.exceptions.ExceptionConverter;
-import org.apache.slider.core.restclient.HttpVerb;
-import org.apache.slider.api.types.PingInformation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.ws.rs.core.MediaType;
-import java.io.IOException;
-import java.util.Map;
-
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
-
-/**
- * Implementation of the {@link SliderApplicationApi}
- */
-public class SliderApplicationApiRestClient extends BaseRestClient
- implements SliderApplicationApi {
- private static final Logger log =
- LoggerFactory.getLogger(SliderApplicationApiRestClient.class);
- private WebResource appResource;
-
- /**
- * Create an instance
- * @param jerseyClient jersey client for operations
- * @param appResource resource of application API
- */
- public SliderApplicationApiRestClient(Client jerseyClient,
- WebResource appResource) {
- super(jerseyClient);
- this.appResource = appResource;
- }
-
- /**
- * Create an instance
- * @param jerseyClient jersey client for operations
- * @param appmaster URL of appmaster/proxy to AM
- */
- public SliderApplicationApiRestClient(Client jerseyClient, String appmaster) {
- super(jerseyClient);
- WebResource amResource = jerseyClient.resource(appmaster);
- amResource.type(MediaType.APPLICATION_JSON);
- this.appResource = amResource.path(SLIDER_PATH_APPLICATION);
- }
-
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("SliderApplicationApiRestClient{");
- sb.append("appResource=").append(appResource);
- sb.append('}');
- return sb.toString();
- }
-
- /**
- * Create a resource under the application path
- * @param subpath path under application
- * @return a resource under the application path
- */
- public WebResource applicationResource(String subpath) {
- Preconditions.checkArgument(!StringUtils.isEmpty(subpath),
- "empty path");
- Preconditions.checkNotNull(appResource, "Null app resource");
- return appResource.path(subpath);
- }
-
- /**
- * Get operation against a path under the Application
- * @param <T> type expected
- * @param subpath path
- * @param c class to instantiate
- * @return instance
- * @throws IOException on any problem
- */
- public <T> T getApplicationResource(String subpath, Class<T> c)
- throws IOException {
- return appResourceOperation(HttpVerb.GET, subpath, c);
- }
-
- /**
- * Get operation against a path under the Application
- * @param <T> type expected
- * @param subpath path
- * @param t type info
- * @return instance
- * @throws IOException on any problem
- */
- public <T> T getApplicationResource(String subpath, GenericType<T> t)
- throws IOException {
- return appResourceOperation(HttpVerb.GET, subpath, t);
- }
-
- /**
- *
- * @param method method to exec
- * @param <T> type expected
- * @param subpath path
- * @param c class to instantiate
- * @return instance
- * @throws IOException on any problem
- */
- public <T> T appResourceOperation(HttpVerb method, String subpath, Class<T> c)
- throws IOException {
- return exec(method, applicationResource(subpath), c);
- }
-
-
- /**
- * Get operation against a path under the Application
- * @param <T> type expected
- * @param subpath path
- * @param t type info
- * @return instance
- * @throws IOException on any problem
- */
- public <T> T appResourceOperation(HttpVerb method, String subpath,
- GenericType<T> t)
- throws IOException {
- return exec(method, applicationResource(subpath), t);
- }
-
-
- @Override
- public AggregateConf getDesiredModel() throws IOException {
- return getApplicationResource(MODEL_DESIRED, AggregateConf.class);
- }
-
- @Override
- public ConfTreeOperations getDesiredAppconf() throws IOException {
- ConfTree resource =
- getApplicationResource(MODEL_DESIRED_APPCONF, ConfTree.class);
- return new ConfTreeOperations(resource);
- }
-
- @Override
- public ConfTreeOperations getDesiredResources() throws IOException {
- ConfTree resource =
- getApplicationResource(MODEL_DESIRED_RESOURCES, ConfTree.class);
- return new ConfTreeOperations(resource);
- }
-
- @Override
- public AggregateConf getResolvedModel() throws IOException {
- return getApplicationResource(MODEL_RESOLVED, AggregateConf.class);
- }
-
-
- @Override
- public ConfTreeOperations getResolvedAppconf() throws IOException {
- ConfTree resource =
- getApplicationResource(MODEL_RESOLVED_APPCONF, ConfTree.class);
- return new ConfTreeOperations(resource);
- }
-
- @Override
- public ConfTreeOperations getResolvedResources() throws IOException {
- ConfTree resource =
- getApplicationResource(MODEL_RESOLVED_RESOURCES, ConfTree.class);
- return new ConfTreeOperations(resource);
- }
-
- @Override
- public ConfTreeOperations getLiveResources() throws IOException {
- ConfTree resource =
- getApplicationResource(LIVE_RESOURCES, ConfTree.class);
- return new ConfTreeOperations(resource);
- }
-
- @Override
- public Map<String, ContainerInformation> enumContainers() throws
- IOException {
- return getApplicationResource(LIVE_CONTAINERS,
- new GenericType<Map<String, ContainerInformation>>() {
- });
- }
-
- @Override
- public ContainerInformation getContainer(String containerId) throws
- IOException {
- return getApplicationResource(LIVE_CONTAINERS + "/" + containerId,
- ContainerInformation.class);
- }
-
- @Override
- public Map<String, ComponentInformation> enumComponents() throws
- IOException {
- return getApplicationResource(LIVE_COMPONENTS,
- new GenericType<Map<String, ComponentInformation>>() { });
- }
-
- @Override
- public ComponentInformation getComponent(String componentName) throws
- IOException {
- return getApplicationResource(LIVE_COMPONENTS + "/" + componentName,
- ComponentInformation.class);
- }
-
- @Override
- public NodeInformationList getLiveNodes() throws IOException {
- return getApplicationResource(LIVE_NODES, NodeInformationList.class);
- }
-
- @Override
- public NodeInformation getLiveNode(String hostname) throws IOException {
- return getApplicationResource(LIVE_NODES + "/" + hostname,
- NodeInformation.class);
- }
-
- @Override
- public PingInformation ping(String text) throws IOException {
- return pingPost(text);
- }
-
- /**
- * Ping as a GET
- * @param text text to include
- * @return the response
- * @throws IOException on any failure
- */
- public PingInformation pingGet(String text) throws IOException {
- WebResource pingResource = applicationResource(ACTION_PING);
- pingResource.getUriBuilder().queryParam("body", text);
- return pingResource.get(PingInformation.class);
- }
-
- /**
- * Ping as a POST
- * @param text text to include
- * @return the response
- * @throws IOException on any failure
- */
- public PingInformation pingPost(String text) throws IOException {
- WebResource pingResource = applicationResource(ACTION_PING);
- Form f = new Form();
- f.add("text", text);
- return pingResource
- .type(MediaType.APPLICATION_JSON_TYPE)
- .post(PingInformation.class, f);
- }
-
- @Override
- public void stop(String text) throws IOException {
- WebResource resource = applicationResource(ACTION_STOP);
- resource.post(text);
- }
-
- @Override
- public ApplicationLivenessInformation getApplicationLiveness() throws IOException {
- return getApplicationResource(LIVE_LIVENESS,
- ApplicationLivenessInformation.class);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
index 2d471ed..a2d4e38 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
@@ -19,17 +19,6 @@
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParametersDelegate;
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.fs.Path;
-import org.apache.slider.core.conf.ConfTree;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.providers.SliderProviderFactory;
-
-import java.io.File;
-import java.util.List;
-import java.util.Map;
/**
* Abstract Action to build things; shares args across build and
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index 5919312..5f5e611 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -237,80 +237,6 @@ public class CoreFileSystem {
}
/**
- * Build up the path string for cluster resource install location -no
- * attempt to create the directory is made
- *
- * @return the path for resource
- */
- public Path buildClusterResourcePath(String clusterName, String component) {
- Preconditions.checkNotNull(clusterName);
- Path path = buildClusterDirPath(clusterName);
- return new Path(path, SliderKeys.RESOURCE_DIR + "/" + component);
- }
-
- /**
- * Build up the path string for cluster resource install location -no
- * attempt to create the directory is made
- *
- * @return the path for resource
- */
- public Path buildClusterResourcePath(String clusterName) {
- Preconditions.checkNotNull(clusterName);
- Path path = buildClusterDirPath(clusterName);
- return new Path(path, SliderKeys.RESOURCE_DIR);
- }
-
- /**
- * Create the Slider cluster path for a named cluster and all its subdirs
- * This is a directory; a mkdirs() operation is executed
- * to ensure that it is there.
- *
- * @param clustername name of the cluster
- * @return the path to the cluster directory
- * @throws java.io.IOException trouble
- * @throws SliderException slider-specific exceptions
- */
- public Path createClusterDirectories(String clustername, Configuration conf)
- throws IOException, SliderException {
-
-
- Path clusterDirectory = buildClusterDirPath(clustername);
- InstancePaths instancePaths = new InstancePaths(clusterDirectory);
- createClusterDirectories(instancePaths);
- return clusterDirectory;
- }
-
- /**
- * Create the Slider cluster path for a named cluster and all its subdirs
- * This is a directory; a mkdirs() operation is executed
- * to ensure that it is there.
- *
- * @param instancePaths instance paths
- * @throws IOException trouble
- * @throws SliderException slider-specific exceptions
- */
- public void createClusterDirectories(InstancePaths instancePaths) throws
- IOException, SliderException {
- Path instanceDir = instancePaths.instanceDir;
-
- verifyDirectoryNonexistent(instanceDir);
- FsPermission clusterPerms = getInstanceDirectoryPermissions();
- createWithPermissions(instanceDir, clusterPerms);
- createWithPermissions(instancePaths.snapshotConfPath, clusterPerms);
- createWithPermissions(instancePaths.generatedConfPath, clusterPerms);
- createWithPermissions(instancePaths.historyPath, clusterPerms);
- createWithPermissions(instancePaths.tmpPathAM, clusterPerms);
-
- // Data Directory
- String dataOpts =
- configuration.get(SliderXmlConfKeys.DATA_DIRECTORY_PERMISSIONS,
- SliderXmlConfKeys.DEFAULT_DATA_DIRECTORY_PERMISSIONS);
- log.debug("Setting data directory permissions to {}", dataOpts);
- createWithPermissions(instancePaths.dataPath, new FsPermission(dataOpts));
-
- }
-
- /**
* Create a directory with the given permissions.
*
* @param dir directory
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index c0ef2d4..9d7c40a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -19,7 +19,6 @@
package org.apache.slider.common.tools;
import com.google.common.base.Preconditions;
-
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
@@ -52,23 +51,18 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.Slider;
-import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.RoleKeys;
import org.apache.slider.api.types.ContainerInformation;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.params.Arguments;
import org.apache.slider.common.params.SliderActions;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.exceptions.BadClusterStateException;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.ErrorStrings;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.ClasspathConstructor;
import org.apache.slider.core.main.LauncherExitCodes;
-import org.apache.slider.providers.agent.AgentKeys;
import org.apache.slider.server.services.utility.PatternValidator;
import org.apache.slider.server.services.workflow.ForkedProcessService;
import org.apache.zookeeper.server.util.KerberosUtil;
@@ -81,7 +75,6 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
-import java.io.FileReader;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
@@ -112,7 +105,6 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.TimeZone;
-import java.util.Timer;
import java.util.TimerTask;
import java.util.TreeMap;
import java.util.TreeSet;
@@ -1778,61 +1770,6 @@ public final class SliderUtils {
VersionInfo.getBranch() + " @" + VersionInfo.getSrcChecksum());
}
- /**
- * Set the time for an information (human, machine) timestamp pair of fields.
- * The human time is the time in millis converted via the {@link Date} class.
- * @param info info fields
- * @param keyHumanTime name of human time key
- * @param keyMachineTime name of machine time
- * @param time timestamp
- */
- public static void setInfoTime(Map info,
- String keyHumanTime,
- String keyMachineTime,
- long time) {
- info.put(keyHumanTime, SliderUtils.toGMTString(time));
- info.put(keyMachineTime, Long.toString(time));
- }
-
- public static Path extractImagePath(CoreFileSystem fs,
- MapOperations internalOptions) throws
- SliderException, IOException {
- Path imagePath;
- String imagePathOption =
- internalOptions.get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
- String appHomeOption =
- internalOptions.get(InternalKeys.INTERNAL_APPLICATION_HOME);
- if (!isUnset(imagePathOption)) {
- if (!isUnset(appHomeOption)) {
- throw new BadClusterStateException(
- ErrorStrings.E_BOTH_IMAGE_AND_HOME_DIR_SPECIFIED);
- }
- imagePath = fs.createPathThatMustExist(imagePathOption);
- } else {
- imagePath = null;
- if (isUnset(appHomeOption)) {
- throw new BadClusterStateException(
- ErrorStrings.E_NO_IMAGE_OR_HOME_DIR_SPECIFIED);
- }
- }
- return imagePath;
- }
-
- /**
- * trigger a JVM halt with no clean shutdown at all
- * @param status status code for exit
- * @param text text message
- * @param delay delay in millis
- * @return the timer (assuming the JVM hasn't halted yet)
- *
- */
- public static Timer haltAM(int status, String text, int delay) {
-
- Timer timer = new Timer("halt timer", false);
- timer.schedule(new DelayedHalt(status, text), delay);
- return timer;
- }
-
public static String propertiesToString(Properties props) {
TreeSet<String> keys = new TreeSet<>(props.stringPropertyNames());
StringBuilder builder = new StringBuilder();
@@ -2379,40 +2316,6 @@ public final class SliderUtils {
}
/**
- * return the HDFS path where the application package has been uploaded
- * manually or by using slider client (install package command)
- *
- * @param conf configuration
- * @return
- */
- public static String getApplicationDefinitionPath(ConfTreeOperations conf)
- throws BadConfigException {
- return getApplicationDefinitionPath(conf, null);
- }
-
- /**
- * return the HDFS path where the application package has been uploaded
- * manually or by using slider client (install package command)
- *
- * @param conf configuration
- * @param roleGroup name of component
- * @return
- */
- public static String getApplicationDefinitionPath(ConfTreeOperations conf,
- String roleGroup)
- throws BadConfigException {
- String appDefPath = conf.getGlobalOptions().getMandatoryOption(
- AgentKeys.APP_DEF);
- if (roleGroup != null) {
- MapOperations component = conf.getComponent(roleGroup);
- if (component != null) {
- appDefPath = component.getOption(AgentKeys.APP_DEF, appDefPath);
- }
- }
- return appDefPath;
- }
-
- /**
* return the path to the slider-client.xml used by the current running
* slider command
*
@@ -2487,25 +2390,6 @@ public final class SliderUtils {
}
/**
- * return a description of whether the current user has created credential
- * cache files from kerberos servers
- *
- * @throws IOException
- * @throws BadConfigException
- * @throws SecurityException
- * - if a security manager exists and its checkPropertyAccess
- * method doesn't allow access to the specified system property.
- */
- public static String checkCredentialCacheFile() throws IOException,
- BadConfigException {
- String result = null;
- if (!Shell.WINDOWS) {
- result = Shell.execCommand("klist");
- }
- return result;
- }
-
- /**
* Compare the times of two applications: most recent app comes first
* Specifically: the one whose start time value is greater.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java
deleted file mode 100644
index f14a07a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java
+++ /dev/null
@@ -1,520 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.buildutils;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.slider.api.InternalKeys;
-import org.apache.slider.api.OptionKeys;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.StatusKeys;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.SliderXmlConfKeys;
-import org.apache.slider.common.tools.CoreFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.conf.MapOperations;
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.ErrorStrings;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.persist.ConfPersister;
-import org.apache.slider.core.persist.InstancePaths;
-import org.apache.slider.core.persist.LockAcquireFailedException;
-import org.apache.slider.core.persist.LockHeldAction;
-import org.apache.slider.core.zk.ZKPathBuilder;
-import org.apache.slider.core.zk.ZookeeperUtils;
-import org.apache.slider.providers.agent.AgentKeys;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeSet;
-
-import static org.apache.slider.api.InternalKeys.INTERNAL_ADDONS_DIR_PATH;
-import static org.apache.slider.api.InternalKeys.INTERNAL_APPDEF_DIR_PATH;
-import static org.apache.slider.api.InternalKeys.INTERNAL_QUEUE;
-import static org.apache.slider.api.OptionKeys.INTERNAL_AM_TMP_DIR;
-import static org.apache.slider.api.OptionKeys.INTERNAL_TMP_DIR;
-import static org.apache.slider.api.OptionKeys.INTERNAL_APPLICATION_HOME;
-import static org.apache.slider.api.OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH;
-import static org.apache.slider.api.OptionKeys.INTERNAL_DATA_DIR_PATH;
-import static org.apache.slider.api.OptionKeys.INTERNAL_GENERATED_CONF_PATH;
-import static org.apache.slider.api.OptionKeys.INTERNAL_SNAPSHOT_CONF_PATH;
-import static org.apache.slider.api.OptionKeys.ZOOKEEPER_HOSTS;
-import static org.apache.slider.api.OptionKeys.ZOOKEEPER_PATH;
-import static org.apache.slider.api.OptionKeys.ZOOKEEPER_QUORUM;
-import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
-import static org.apache.slider.common.SliderKeys.COMPONENT_AM;
-import static org.apache.slider.common.SliderKeys.COMPONENT_SEPARATOR;
-import static org.apache.slider.common.SliderKeys.COMPONENT_TYPE_EXTERNAL_APP;
-import static org.apache.slider.common.SliderKeys.COMPONENT_TYPE_KEY;
-import static org.apache.slider.common.tools.SliderUtils.isClusternameValid;
-
-/**
- * Build up the instance of a cluster.
- */
-public class InstanceBuilder {
-
- private final String clustername;
- private final Configuration conf;
- private final CoreFileSystem coreFS;
- private final InstancePaths instancePaths;
- private AggregateConf instanceDescription;
- private Map<String, Path> externalAppDefs = new HashMap<>();
- private TreeSet<Integer> priorities = new TreeSet<>();
-
- private static final Logger log =
- LoggerFactory.getLogger(InstanceBuilder.class);
-
- public InstanceBuilder(CoreFileSystem coreFileSystem,
- Configuration conf,
- String clustername) {
- this.clustername = clustername;
- this.conf = conf;
- this.coreFS = coreFileSystem;
- Path instanceDir = coreFileSystem.buildClusterDirPath(clustername);
- instancePaths = new InstancePaths(instanceDir);
-
- }
-
- public AggregateConf getInstanceDescription() {
- return instanceDescription;
- }
-
- public InstancePaths getInstancePaths() {
- return instancePaths;
- }
-
-
- @Override
- public String toString() {
- return "Builder working with " + clustername + " at " +
- getInstanceDir();
- }
-
- private Path getInstanceDir() {
- return instancePaths.instanceDir;
- }
-
- /**
- * Initial part of the build process
- * @param instanceConf
- * @param provider
- */
- public void init(
- String provider,
- AggregateConf instanceConf) {
-
-
- this.instanceDescription = instanceConf;
-
- //internal is extended
- ConfTreeOperations internalOps = instanceConf.getInternalOperations();
-
- Map<String, Object> md = internalOps.getConfTree().metadata;
- long time = System.currentTimeMillis();
- md.put(StatusKeys.INFO_CREATE_TIME_HUMAN, SliderUtils.toGMTString(time));
- md.put(StatusKeys.INFO_CREATE_TIME_MILLIS, Long.toString(time));
-
- BuildHelper.addBuildMetadata(md, "create");
- SliderUtils.setInfoTime(md,
- StatusKeys.INFO_CREATE_TIME_HUMAN,
- StatusKeys.INFO_CREATE_TIME_MILLIS,
- System.currentTimeMillis());
-
- internalOps.set(INTERNAL_AM_TMP_DIR,
- instancePaths.tmpPathAM.toUri());
- internalOps.set(INTERNAL_TMP_DIR,
- instancePaths.tmpPath.toUri());
- internalOps.set(INTERNAL_SNAPSHOT_CONF_PATH,
- instancePaths.snapshotConfPath.toUri());
- internalOps.set(INTERNAL_GENERATED_CONF_PATH,
- instancePaths.generatedConfPath.toUri());
- internalOps.set(INTERNAL_DATA_DIR_PATH,
- instancePaths.dataPath.toUri());
- internalOps.set(INTERNAL_APPDEF_DIR_PATH,
- instancePaths.appDefPath.toUri());
- internalOps.set(INTERNAL_ADDONS_DIR_PATH,
- instancePaths.addonsPath.toUri());
-
-
- internalOps.set(InternalKeys.INTERNAL_PROVIDER_NAME, provider);
- internalOps.set(OptionKeys.APPLICATION_NAME, clustername);
-
- }
-
- /**
- * Set the queue used to start the application
- * @param queue
- * @throws BadConfigException
- */
- public void setQueue(String queue) throws BadConfigException {
- if(queue != null) {
- if(SliderUtils.isUnset(queue)) {
- throw new BadConfigException("Queue value cannot be empty.");
- }
-
- instanceDescription.getInternalOperations().set(INTERNAL_QUEUE, queue);
- }
- }
-
- /**
- * Set up the image/app home path
- * @param appImage path in the DFS to the tar file
- * @param appHomeDir other strategy: home dir
- * @throws BadConfigException if both are found
- */
- public void setImageDetailsIfAvailable(
- Path appImage,
- String appHomeDir) throws BadConfigException {
- boolean appHomeUnset = SliderUtils.isUnset(appHomeDir);
- // App home or image
- if (appImage != null) {
- if (!appHomeUnset) {
- // both args have been set
- throw new BadConfigException(
- ErrorStrings.E_BOTH_IMAGE_AND_HOME_DIR_SPECIFIED);
- }
- instanceDescription.getInternalOperations().set(INTERNAL_APPLICATION_IMAGE_PATH,
- appImage.toUri());
- } else {
- // the alternative is app home, which now MUST be set
- if (!appHomeUnset) {
- instanceDescription.getInternalOperations().set(INTERNAL_APPLICATION_HOME,
- appHomeDir);
- }
- }
- }
-
-
- /**
- * Propagate any critical principals from the current site config down to the HBase one.
- */
- public void propagatePrincipals() {
- String dfsPrincipal = conf.get(SliderXmlConfKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
- if (dfsPrincipal != null) {
- String siteDfsPrincipal = OptionKeys.SITE_XML_PREFIX +
- SliderXmlConfKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
- instanceDescription.getAppConfOperations().set(siteDfsPrincipal, dfsPrincipal);
- }
- }
-
- public void propagateFilename() {
- String fsDefaultName = conf.get(
- CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY);
- instanceDescription.getAppConfOperations().set(OptionKeys.SITE_XML_PREFIX +
- CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
- fsDefaultName
- );
-
- instanceDescription.getAppConfOperations().set(OptionKeys.SITE_XML_PREFIX +
- SliderXmlConfKeys.FS_DEFAULT_NAME_CLASSIC,
- fsDefaultName
- );
-
- }
-
-
- public void takeSnapshotOfConfDir(Path appconfdir) throws
- IOException,
- BadConfigException,
- BadClusterStateException {
- FileSystem srcFS = FileSystem.get(appconfdir.toUri(), conf);
- if (!srcFS.isDirectory(appconfdir)) {
- throw new BadConfigException(
- "Source Configuration directory is not valid: %s",
- appconfdir.toString());
- }
- // bulk copy
- FsPermission clusterPerms = coreFS.getInstanceDirectoryPermissions();
- // first the original from wherever to the DFS
- SliderUtils.copyDirectory(conf, appconfdir, instancePaths.snapshotConfPath,
- clusterPerms);
- }
-
-
- private Set<String> getExternalComponents(ConfTreeOperations ops)
- throws BadConfigException {
- Set<String> externalComponents = new HashSet<>();
- if (ops.getGlobalOptions().containsKey(COMPONENT_TYPE_KEY)) {
- throw new BadConfigException(COMPONENT_TYPE_KEY + " must be " +
- "specified per-component, not in global");
- }
-
- for (Entry<String, Map<String, String>> entry : ops.getComponents()
- .entrySet()) {
- if (COMPONENT_AM.equals(entry.getKey())) {
- continue;
- }
- Map<String, String> options = entry.getValue();
- if (COMPONENT_TYPE_EXTERNAL_APP.equals(options.get(COMPONENT_TYPE_KEY))) {
- externalComponents.add(entry.getKey());
- }
- }
- return externalComponents;
- }
-
- private void mergeExternalComponent(ConfTreeOperations ops,
- ConfTreeOperations externalOps, String externalComponent,
- Integer priority) throws BadConfigException {
- for (String subComponent : externalOps.getComponentNames()) {
- if (COMPONENT_AM.equals(subComponent)) {
- continue;
- }
- String prefix = externalComponent + COMPONENT_SEPARATOR;
- log.debug("Merging options for {} into {}", subComponent,
- prefix + subComponent);
- MapOperations subComponentOps = ops.getOrAddComponent(
- prefix + subComponent);
- if (priority == null) {
- SliderUtils.mergeMaps(subComponentOps,
- ops.getComponent(externalComponent).options);
- subComponentOps.remove(COMPONENT_TYPE_KEY);
- }
-
- SliderUtils.mergeMapsIgnoreDuplicateKeysAndPrefixes(subComponentOps,
- externalOps.getComponent(subComponent),
- SliderKeys.COMPONENT_KEYS_TO_SKIP);
-
- // add prefix to existing prefix
- String existingPrefix = subComponentOps.get(ROLE_PREFIX);
- if (existingPrefix != null) {
- if (!subComponent.startsWith(existingPrefix)) {
- throw new BadConfigException("Bad prefix " + existingPrefix +
- " for subcomponent " + subComponent + " of " + externalComponent);
- }
- prefix = prefix + existingPrefix;
- }
- subComponentOps.set(ROLE_PREFIX, prefix);
-
- // adjust priority
- if (priority != null) {
- subComponentOps.put(ResourceKeys.COMPONENT_PRIORITY,
- Integer.toString(priority));
- priorities.add(priority);
- priority++;
- }
- }
- }
-
- private int getNextPriority() {
- if (priorities.isEmpty()) {
- return 1;
- } else {
- return priorities.last() + 1;
- }
- }
-
- public void resolve()
- throws BadConfigException, IOException, BadClusterStateException {
- ConfTreeOperations appConf = instanceDescription.getAppConfOperations();
- ConfTreeOperations resources = instanceDescription.getResourceOperations();
-
- for (Entry<String, Map<String, String>> entry : resources.getComponents()
- .entrySet()) {
- if (COMPONENT_AM.equals(entry.getKey())) {
- continue;
- }
- if (entry.getValue().containsKey(ResourceKeys.COMPONENT_PRIORITY)) {
- priorities.add(Integer.parseInt(entry.getValue().get(
- ResourceKeys.COMPONENT_PRIORITY)));
- }
- }
-
- Set<String> externalComponents = getExternalComponents(appConf);
- if (!externalComponents.isEmpty()) {
- log.info("Found external components {}", externalComponents);
- }
-
- for (String component : externalComponents) {
- if (!isClusternameValid(component)) {
- throw new BadConfigException(component + " is not a valid external " +
- "component");
- }
- Path componentClusterDir = coreFS.buildClusterDirPath(component);
- try {
- coreFS.verifyPathExists(componentClusterDir);
- } catch (IOException e) {
- throw new BadConfigException("external component " + component +
- " doesn't exist");
- }
- AggregateConf componentConf = new AggregateConf();
- ConfPersister persister = new ConfPersister(coreFS, componentClusterDir);
- try {
- persister.load(componentConf);
- } catch (Exception e) {
- throw new BadConfigException("Couldn't read configuration for " +
- "external component " + component);
- }
-
- ConfTreeOperations componentAppConf = componentConf.getAppConfOperations();
- String externalAppDef = componentAppConf.get(AgentKeys.APP_DEF);
- if (SliderUtils.isSet(externalAppDef)) {
- Path newAppDef = new Path(coreFS.buildAppDefDirPath(clustername),
- component + "_" + SliderKeys.DEFAULT_APP_PKG);
- componentAppConf.set(AgentKeys.APP_DEF, newAppDef);
- componentAppConf.append(AgentKeys.APP_DEF_ORIGINAL, externalAppDef);
- log.info("Copying external appdef {} to {} for {}", externalAppDef,
- newAppDef, component);
- externalAppDefs.put(externalAppDef, newAppDef);
- externalAppDef = newAppDef.toString();
- }
-
- for (String rcomp : componentConf.getResourceOperations()
- .getComponentNames()) {
- if (COMPONENT_AM.equals(rcomp)) {
- continue;
- }
- log.debug("Adding component {} to appConf for {}", rcomp, component);
- componentAppConf.getOrAddComponent(rcomp);
- }
- componentConf.resolve();
-
- for (String rcomp : componentConf.getResourceOperations()
- .getComponentNames()) {
- if (COMPONENT_AM.equals(rcomp)) {
- continue;
- }
- String componentAppDef = componentAppConf.getComponentOpt(
- rcomp, AgentKeys.APP_DEF, null);
- if (SliderUtils.isUnset(componentAppDef) ||
- componentAppDef.equals(externalAppDef)) {
- continue;
- }
- if (externalAppDefs.containsKey(componentAppDef)) {
- log.info("Using external appdef {} for {}",
- externalAppDefs.get(componentAppDef), rcomp);
- } else {
- String existingPrefix = componentAppConf.getComponentOpt(rcomp,
- ROLE_PREFIX, null);
- if (SliderUtils.isUnset(existingPrefix)) {
- existingPrefix = "";
- } else {
- existingPrefix = COMPONENT_SEPARATOR + SliderUtils.trimPrefix(
- existingPrefix);
- }
- Path newAppDef = new Path(coreFS.buildAppDefDirPath(clustername),
- component + existingPrefix + "_" + SliderKeys.DEFAULT_APP_PKG);
- externalAppDefs.put(componentAppDef, newAppDef);
- log.info("Copying external appdef {} to {} for {}", componentAppDef,
- newAppDef, component + COMPONENT_SEPARATOR + rcomp);
- }
- componentAppConf.setComponentOpt(rcomp, AgentKeys.APP_DEF,
- externalAppDefs.get(componentAppDef).toString());
- componentAppConf.appendComponentOpt(rcomp,
- AgentKeys.APP_DEF_ORIGINAL, componentAppDef);
- }
- Set<Path> newAppDefs = new HashSet<>();
- newAppDefs.addAll(externalAppDefs.values());
- if (newAppDefs.size() != externalAppDefs.size()) {
- throw new IllegalStateException("Values repeat in external appdefs "
- + externalAppDefs);
- }
- log.info("External appdefs after {}: {}", component, externalAppDefs);
-
- SliderUtils.mergeMapsIgnoreDuplicateKeys(
- appConf.getConfTree().credentials,
- componentAppConf.getConfTree().credentials);
-
- mergeExternalComponent(appConf, componentAppConf, component, null);
- mergeExternalComponent(resources, componentConf.getResourceOperations(),
- component, getNextPriority());
- }
- }
-
-
- /**
- * Persist this
- * @param appconfdir conf dir
- * @param overwrite if true, we don't need to create cluster dir
- * @throws IOException
- * @throws SliderException
- * @throws LockAcquireFailedException
- */
- public void persist(Path appconfdir, boolean overwrite) throws
- IOException,
- SliderException,
- LockAcquireFailedException {
- if (!overwrite) {
- coreFS.createClusterDirectories(instancePaths);
- }
- ConfPersister persister =
- new ConfPersister(coreFS, getInstanceDir());
- ConfDirSnapshotAction action = null;
- if (appconfdir != null) {
- action = new ConfDirSnapshotAction(appconfdir);
- }
- persister.save(instanceDescription, action);
- for (Entry<String, Path> appDef : externalAppDefs.entrySet()) {
- SliderUtils.copy(conf, new Path(appDef.getKey()), appDef.getValue());
- }
- }
-
- /**
- * Add the ZK paths to the application options.
- *
- * @param zkBinding ZK binding
- */
- public void addZKBinding(ZKPathBuilder zkBinding) throws BadConfigException {
-
- String quorum = zkBinding.getAppQuorum();
- if (SliderUtils.isSet(quorum)) {
- MapOperations globalAppOptions =
- instanceDescription.getAppConfOperations().getGlobalOptions();
- globalAppOptions.put(ZOOKEEPER_PATH, zkBinding.getAppPath());
- globalAppOptions.put(ZOOKEEPER_QUORUM, quorum);
- globalAppOptions.put(ZOOKEEPER_HOSTS,
- ZookeeperUtils.convertToHostsOnlyList(quorum));
- }
- }
-
- /**
- * Class to execute the snapshotting of the configuration directory
- * while the persistence lock is held.
- *
- * This guarantees that there won't be an attempt to launch a cluster
- * until the snapshot is complete -as the write lock won't be released
- * until afterwards.
- */
- private class ConfDirSnapshotAction implements LockHeldAction {
-
- private final Path appconfdir;
-
- private ConfDirSnapshotAction(Path appconfdir) {
- this.appconfdir = appconfdir;
- }
-
- @Override
- public void execute() throws IOException, SliderException {
-
- takeSnapshotOfConfDir(appconfdir);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceIO.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceIO.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceIO.java
deleted file mode 100644
index 262b6ee..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceIO.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.buildutils;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.slider.common.tools.CoreFileSystem;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.persist.ConfPersister;
-import org.apache.slider.core.persist.LockAcquireFailedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-public class InstanceIO {
- protected static final Logger log =
- LoggerFactory.getLogger(InstanceIO.class);
-
- /**
- * Load in an instance definition -but do not resolve it
- * @param sliderFileSystem filesystem
- * @param clusterDirectory CD
- * @return the unresolved aggregate configuration
- * @throws IOException
- * @throws SliderException
- * @throws BadClusterStateException if a lock could not be acquired
- */
- public static AggregateConf loadInstanceDefinitionUnresolved(
- CoreFileSystem sliderFileSystem,
- Path clusterDirectory)
- throws IOException, SliderException {
- AggregateConf instanceDefinition = new AggregateConf();
- ConfPersister persister =
- new ConfPersister(sliderFileSystem, clusterDirectory);
- try {
- persister.load(instanceDefinition);
- } catch (LockAcquireFailedException e) {
- log.debug("Lock acquisition failure of {}", clusterDirectory, e);
-
- throw new BadClusterStateException(
- "Application at %s is locked for reading",
- clusterDirectory.toString());
- }
- return instanceDefinition;
- }
-
-
- /**
- * Update a persisted instance definition
- * @param coreFS filesystem
- * @param dir directory to load from
- * @param instanceDefinition instance definition to save do
- * @throws SliderException
- * @throws IOException
- * @throws LockAcquireFailedException
- */
- public static void saveInstanceDefinition(CoreFileSystem coreFS,
- Path dir,
- AggregateConf instanceDefinition)
- throws SliderException, IOException, LockAcquireFailedException {
- ConfPersister persister =
- new ConfPersister(coreFS, dir);
- persister.save(instanceDefinition, null);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AbstractInputPropertiesValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AbstractInputPropertiesValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AbstractInputPropertiesValidator.java
deleted file mode 100644
index 336b4dc..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AbstractInputPropertiesValidator.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.core.conf;
-
-import org.apache.slider.core.exceptions.BadConfigException;
-
-/**
- *
- */
-public abstract class AbstractInputPropertiesValidator implements InputPropertiesValidator{
-
- abstract void validatePropertyNamePrefix(String key) throws BadConfigException;
-
- public void validate(ConfTreeOperations props)
- throws BadConfigException {
- validateGlobalProperties(props);
- validateComponentProperties(props);
-
- }
-
- protected void validateComponentProperties(ConfTreeOperations props)
- throws BadConfigException {
- for (String compName : props.getComponentNames()) {
- MapOperations mo = props.getComponent(compName);
- if (mo == null) continue;
- for (String key : mo.keySet()) {
- validatePropertyNamePrefix(key);
- }
- }
- }
-
- abstract void validateGlobalProperties(ConfTreeOperations props)
- throws BadConfigException;
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
deleted file mode 100644
index a272420..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.conf;
-
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.codehaus.jackson.annotate.JsonIgnore;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.io.IOException;
-
-/**
- * Aggregate Configuration.
- *
- * It is serializable to JSON
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public final class AggregateConf {
-
- private String name;
- private ConfTree resources;
- private ConfTree internal;
- private ConfTree appConf;
-
- private ConfTreeOperations resourceOperations;
- private ConfTreeOperations appConfOperations;
- private ConfTreeOperations internalOperations;
-
- private String passphrase;
-
- public AggregateConf() {
- this(new ConfTree(), new ConfTree(), new ConfTree());
- }
-
- public AggregateConf(String name) {
- this(new ConfTree(), new ConfTree(), new ConfTree());
- this.name = name;
- }
-
- public AggregateConf(ConfTree resources,
- ConfTree appConf,
- ConfTree internal) {
- setResources(resources);
- setAppConf(appConf);
- setInternal(internal);
- }
-
- /**
- * Take a snapshot of the configuration
- * @param instanceDefinition source
- * @throws IOException marshalling/copying problems
- */
- public AggregateConf(AggregateConf instanceDefinition) throws IOException {
- ConfTreeOperations resourcesSnapshot =
- ConfTreeOperations.fromInstance(instanceDefinition.getResources());
- ConfTreeOperations appConfSnapshot =
- ConfTreeOperations.fromInstance(instanceDefinition.getAppConf());
- ConfTreeOperations internalsSnapshot =
- ConfTreeOperations.fromInstance(instanceDefinition.getInternal());
- //build a new aggregate from the snapshots
- setResources(resourcesSnapshot.confTree);
- setAppConf(appConfSnapshot.confTree);
- setInternal(internalsSnapshot.confTree);
- }
-
- public void setResources(ConfTree resources) {
- this.resources = resources;
- resourceOperations = new ConfTreeOperations(resources);
- }
-
- public void setAppConf(ConfTree appConf) {
- this.appConf = appConf;
- appConfOperations = new ConfTreeOperations(appConf);
- }
-
- public ConfTree getInternal() {
- return internal;
- }
-
- public void setInternal(ConfTree internal) {
- this.internal = internal;
- internalOperations = new ConfTreeOperations(internal);
- }
-
- public ConfTree getResources() {
- return resources;
- }
-
- public ConfTree getAppConf() {
- return appConf;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- @JsonIgnore
- public ConfTreeOperations getResourceOperations() {
- return resourceOperations;
- }
-
-
- @JsonIgnore
- public ConfTreeOperations getAppConfOperations() {
- return appConfOperations;
- }
-
- @JsonIgnore
- public ConfTreeOperations getInternalOperations() {
- return internalOperations;
- }
-
- /**
- * predicate to query if all sections have data structures
- * @return true if every section is non-null
- */
- @JsonIgnore
- public boolean isComplete() {
- return resources != null && appConf != null && internal != null;
- }
-
- public void validate() throws BadConfigException {
- if (!isComplete()) {
- throw new BadConfigException("Incomplete instance %s", this);
- }
- resourceOperations.validate();
- internalOperations.validate();
- appConfOperations.validate();
- }
-
- public void resolve() throws BadConfigException {
- validate();
- resourceOperations.resolve();
- internalOperations.resolve();
- appConfOperations.resolve();
- }
-
- @JsonIgnore
- public String getPassphrase() {
- if (passphrase == null) {
- passphrase = RandomStringUtils.randomAlphanumeric(
- Integer.parseInt(SliderKeys.PASS_LEN));
- }
-
- return passphrase;
- }
-
- /**
- * Is this app package versioned?
- *
- * @return true if {@link SliderKeys#APP_VERSION} was set in the app config
- * provided during creation of this app
- * @since 0.80.0-incubating
- */
- public boolean isVersioned() {
- return StringUtils.isNotEmpty(getAppConfOperations().getGlobalOptions()
- .get(SliderKeys.APP_VERSION));
- }
-
- /**
- * string operation includes all the inner conftrees
- * @return a string description
- */
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("{");
- sb.append(",\n\"internal\": ").append(internal);
- sb.append(",\n\"resources\": ").append(resources);
- sb.append(",\n\"appConf\" :").append(appConf);
- sb.append('}');
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
deleted file mode 100644
index 74b6abb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.conf;
-
-import org.apache.slider.core.persist.ConfTreeSerDeser;
-import org.apache.slider.core.persist.PersistKeys;
-import org.codehaus.jackson.JsonGenerationException;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.JsonMappingException;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * A conf tree represents one of the configuration trees
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public final class ConfTree {
-
- /**
- * Size of an initial map. This is kept low so the cost of having
- * many conf trees in a process is low.
- */
- public static final int INITAL_MAP_CAPACITY = 3;
-
- protected static final Logger
- log = LoggerFactory.getLogger(ConfTree.class);
-
- /**
- * version counter
- */
- public String schema = PersistKeys.SCHEMA;
-
- /**
- * Metadata
- */
- public Map<String, Object> metadata = new HashMap<>(INITAL_MAP_CAPACITY);
-
-
- /**
- * Global options
- */
- public Map<String, String> global =
- new HashMap<>(INITAL_MAP_CAPACITY);
-
-
- /**
- * Credentials
- */
- public Map<String, List<String>> credentials =
- new HashMap<>(INITAL_MAP_CAPACITY);
-
- /**
- * Role options,
- * role -> option -> value
- */
- public Map<String, Map<String, String>> components =
- new HashMap<>(INITAL_MAP_CAPACITY);
-
- @Override
- public String toString() {
- try {
- return toJson();
- } catch (Exception e) {
- log.warn("Failed to convert to JSON ", e);
- return super.toString();
- }
- }
-
- /**
- * Convert to a JSON string
- * @return a JSON string description
- * @throws IOException Problems mapping/writing the object
- */
- public String toJson() throws IOException {
- return ConfTreeSerDeser.toString(this);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
deleted file mode 100644
index 526e17d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
+++ /dev/null
@@ -1,527 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.conf;
-
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.persist.ConfTreeSerDeser;
-import org.apache.slider.core.persist.PersistKeys;
-import org.codehaus.jackson.JsonGenerationException;
-import org.codehaus.jackson.annotate.JsonIgnore;
-import org.codehaus.jackson.map.JsonMappingException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-public class ConfTreeOperations {
-
- public final ConfTree confTree;
- private final MapOperations globalOptions;
-
- protected static final Logger
- log = LoggerFactory.getLogger(ConfTreeOperations.class);
-
-
- public ConfTreeOperations(ConfTree confTree) {
- assert confTree != null : "null tree";
- assert confTree.components != null : "null tree components";
- this.confTree = confTree;
- globalOptions = new MapOperations("global", confTree.global);
- }
-
- /**
- * Get the underlying conf tree
- * @return the tree
- */
- public ConfTree getConfTree() {
- return confTree;
- }
-
- /**
- * Validate the configuration
- * @throws BadConfigException
- */
- public void validate() throws BadConfigException {
- validate(null);
- }
-
- /**
- * Validate the configuration
- * @param validator a provided properties validator
- * @throws BadConfigException
- */
- public void validate(InputPropertiesValidator validator) throws BadConfigException {
- String version = confTree.schema;
- if (version == null) {
- throw new BadConfigException("'version' undefined");
- }
- if (!PersistKeys.SCHEMA.equals(version)) {
- throw new BadConfigException(
- "version %s incompatible with supported version %s",
- version,
- PersistKeys.SCHEMA);
- }
- if (validator != null) {
- validator.validate(this);
- }
- }
-
- /**
- * Resolve a ConfTree by mapping all global options into each component
- * -if there is none there already
- */
- public void resolve() {
- for (Map.Entry<String, Map<String, String>> comp : confTree.components.entrySet()) {
- mergeInGlobal(comp.getValue());
- }
- }
-
- /**
- * Merge any options
- * @param component dest values
- */
- public void mergeInGlobal(Map<String, String> component) {
- SliderUtils.mergeMapsIgnoreDuplicateKeys(component, confTree.global);
- }
-
- /**
- * Get operations on the global set
- * @return a wrapped map
- */
- public MapOperations getGlobalOptions() {
- return globalOptions;
- }
-
-
- /**
- * look up a component and return its options
- * @param component component name
- * @return component mapping or null
- */
- public MapOperations getComponent(String component) {
- Map<String, String> instance = confTree.components.get(component);
- if (instance != null) {
- return new MapOperations(component, instance);
- }
- return null;
- }
-
- /**
- * look up a component and return its options with the specified replacements
- * @param component component name
- * @param replacementOptions replacement options
- * @return component mapping or null
- */
- public MapOperations getComponent(String component, Map<String,String>
- replacementOptions) {
- Map<String, String> instance = confTree.components.get(component);
- if (instance != null) {
- Map<String, String> newInstance = new HashMap<>();
- newInstance.putAll(instance);
- newInstance.putAll(replacementOptions);
- return new MapOperations(component, newInstance);
- }
- return null;
- }
-
- /**
- * Get at the underlying component map
- * @return a map of components. This is the raw ConfTree data structure
- */
- public Map<String, Map<String, String>> getComponents() {
- return confTree.components;
- }
-
- /**
- * Get a component -adding it to the components map if
- * none with that name exists
- * @param name role
- * @return role mapping
- */
- public MapOperations getOrAddComponent(String name) {
- MapOperations operations = getComponent(name);
- if (operations != null) {
- return operations;
- }
- //create a new instances
- Map<String, String> map = new HashMap<>();
- confTree.components.put(name, map);
- return new MapOperations(name, map);
- }
-
-
- /*
- * return the Set of names names
- */
- @JsonIgnore
- public Set<String> getComponentNames() {
- return new HashSet<String>(confTree.components.keySet());
- }
-
-
-
- /**
- * Get a component whose presence is mandatory
- * @param name component name
- * @return the mapping
- * @throws BadConfigException if the name is not there
- */
- public MapOperations getMandatoryComponent(String name) throws
- BadConfigException {
- MapOperations ops = getComponent(name);
- if (ops == null) {
- throw new BadConfigException("Missing component " + name);
- }
- return ops;
- }
-
- /**
- * Set a global option, converting it to a string as needed
- * @param key key
- * @param value non null value
- */
- public void set(String key, Object value) {
- globalOptions.put(key, value.toString());
- }
- /**
- * get a global option
- * @param key key
- * @return value or null
- *
- */
- public String get(String key) {
- return globalOptions.get(key);
- }
- /**
- * append to a global option
- * @param key key
- * @return value
- *
- */
- public String append(String key, String value) {
- if (SliderUtils.isUnset(value)) {
- return null;
- }
- if (globalOptions.containsKey(key)) {
- globalOptions.put(key, globalOptions.get(key) + "," + value);
- } else {
- globalOptions.put(key, value);
- }
- return globalOptions.get(key);
- }
-
- /**
- * Propagate all global keys matching a prefix
- * @param src source
- * @param prefix prefix
- */
- public void propagateGlobalKeys(ConfTree src, String prefix) {
- Map<String, String> global = src.global;
- for (Map.Entry<String, String> entry : global.entrySet()) {
- String key = entry.getKey();
- if (key.startsWith(prefix)) {
- set(key, entry.getValue());
- }
- }
- }
-
- /**
- * Propagate all global keys matching a prefix
- * @param src source
- * @param prefix prefix
- */
- public void propagateGlobalKeys(ConfTreeOperations src, String prefix) {
- propagateGlobalKeys(src.confTree, prefix);
- }
-
- /**
- * Merge the map of a single component
- * @param component component name
- * @param map map to merge
- */
- public void mergeSingleComponentMap(String component, Map<String, String> map) {
- MapOperations comp = getOrAddComponent(component);
- comp.putAll(map);
- }
- /**
- * Merge the map of a single component
- * @param component component name
- * @param map map to merge
- */
- public void mergeSingleComponentMapPrefix(String component,
- Map<String, String> map,
- String prefix,
- boolean overwrite) {
- boolean needsMerge = false;
- for (Map.Entry<String, String> entry : map.entrySet()) {
- String key = entry.getKey();
- if (key.startsWith(prefix)) {
- needsMerge = true;
- break;
- }
- }
- if (!needsMerge) {
- return;
- }
- MapOperations comp = getOrAddComponent(component);
- comp.mergeMapPrefixedKeys(map,prefix, overwrite);
- }
-
- /**
- * Merge in components
- * @param commandOptions component options on the CLI
- */
- public void mergeComponents(Map<String, Map<String, String>> commandOptions) {
- for (Map.Entry<String, Map<String, String>> entry : commandOptions.entrySet()) {
- mergeSingleComponentMap(entry.getKey(), entry.getValue());
- }
- }
-
- /**
- * Merge in components
- * @param commandOptions component options on the CLI
- */
- public void mergeComponentsPrefix(Map<String,
- Map<String, String>> commandOptions,
- String prefix,
- boolean overwrite) {
- for (Map.Entry<String, Map<String, String>> entry : commandOptions.entrySet()) {
- mergeSingleComponentMapPrefix(entry.getKey(), entry.getValue(), prefix, overwrite);
- }
- }
-
- /**
- * Merge in another tree -no overwrites of global or conf data
- * (note that metadata does a naive putAll merge/overwrite)
- * @param that the other tree
- */
- public void mergeWithoutOverwrite(ConfTree that) {
-
- getGlobalOptions().mergeWithoutOverwrite(that.global);
- confTree.metadata.putAll(that.metadata);
- confTree.credentials.putAll(that.credentials);
-
- for (Map.Entry<String, Map<String, String>> entry : that.components.entrySet()) {
- MapOperations comp = getOrAddComponent(entry.getKey());
- comp.mergeWithoutOverwrite(entry.getValue());
- }
- }
-
- /**
- * Merge in another tree with overwrites
- * @param that the other tree
- */
- public void merge(ConfTree that) {
-
- getGlobalOptions().putAll(that.global);
- confTree.metadata.putAll(that.metadata);
- confTree.credentials.putAll(that.credentials);
-
- for (Map.Entry<String, Map<String, String>> entry : that.components.entrySet()) {
- MapOperations comp = getOrAddComponent(entry.getKey());
- comp.putAll(entry.getValue());
- }
- }
-
-
- /**
- * Load from a resource. The inner conf tree is the loaded data -unresolved
- * @param resource resource
- * @return loaded value
- * @throws IOException load failure
- */
- public static ConfTreeOperations fromResource(String resource) throws
- IOException {
- ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser();
- ConfTreeOperations ops = new ConfTreeOperations(
- confTreeSerDeser.fromResource(resource) );
- return ops;
- }
-
- /**
- * Load from a resource. The inner conf tree is the loaded data -unresolved
- * @param resource resource
- * @return loaded value
- * @throws IOException load failure
- */
- public static ConfTreeOperations fromFile(File resource) throws
- IOException {
- ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser();
- ConfTreeOperations ops = new ConfTreeOperations(
- confTreeSerDeser.fromFile(resource) );
- return ops;
- }
-
- /**
- * Build from an existing instance -which is cloned via JSON ser/deser
- * @param instance the source instance
- * @return loaded value
- * @throws IOException load failure
- */
- public static ConfTreeOperations fromInstance(ConfTree instance) throws
- IOException {
- ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser();
- ConfTreeOperations ops = new ConfTreeOperations(
- confTreeSerDeser.fromJson(confTreeSerDeser.toJson(instance)) );
- return ops;
- }
-
- /**
- * Load from a file and merge it in
- * @param file file
- * @throws IOException any IO problem
- * @throws BadConfigException if the file is invalid
- */
- public void mergeFile(File file) throws IOException, BadConfigException {
- mergeFile(file, null);
- }
-
- /**
- * Load from a file and merge it in
- * @param file file
- * @param validator properties validator
- * @throws IOException any IO problem
- * @throws BadConfigException if the file is invalid
- */
- public void mergeFile(File file, InputPropertiesValidator validator) throws IOException, BadConfigException {
- ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser();
- ConfTree tree = confTreeSerDeser.fromFile(file);
- ConfTreeOperations ops = new ConfTreeOperations(tree);
- ops.validate(validator);
- merge(ops.confTree);
- }
-
- @Override
- public String toString() {
- return confTree.toString();
- }
-
- /**
- * Convert to a JSON string
- * @return a JSON string description
- */
- public String toJson() throws IOException,
- JsonGenerationException,
- JsonMappingException {
- return confTree.toJson();
- }
-
- /**
- * Get a component option
- * @param name component name
- * @param option option name
- * @param defVal default value
- * @return resolved value
- */
- public String getComponentOpt(String name, String option, String defVal) {
- MapOperations roleopts = getComponent(name);
- if (roleopts == null) {
- return defVal;
- }
- return roleopts.getOption(option, defVal);
- }
-
- /**
- * Get a component opt; use {@link Integer#decode(String)} so as to take hex
- * oct and bin values too.
- *
- * @param name component name
- * @param option option name
- * @param defVal default value
- * @return parsed value
- * @throws NumberFormatException if the role could not be parsed.
- */
- public int getComponentOptInt(String name, String option, int defVal) {
- String val = getComponentOpt(name, option, Integer.toString(defVal));
- return Integer.decode(val);
- }
-
- /**
- * Get a component opt as a boolean using {@link Boolean#valueOf(String)}.
- *
- * @param name component name
- * @param option option name
- * @param defVal default value
- * @return parsed value
- * @throws NumberFormatException if the role could not be parsed.
- */
- public boolean getComponentOptBool(String name, String option, boolean defVal) {
- String val = getComponentOpt(name, option, Boolean.toString(defVal));
- return Boolean.valueOf(val);
- }
-
- /**
- * Set a component option, creating the component if necessary
- * @param component component name
- * @param option option name
- * @param val value
- */
- public void setComponentOpt(String component, String option, String val) {
- Map<String, String> roleopts = getOrAddComponent(component);
- roleopts.put(option, val);
- }
-
- /**
- * Set an integer role option, creating the role if necessary
- * @param role role name
- * @param option option name
- * @param val integer value
- */
- public void setComponentOpt(String role, String option, int val) {
- setComponentOpt(role, option, Integer.toString(val));
- }
- /**
- * Set a long role option, creating the role if necessary
- * @param role role name
- * @param option option name
- * @param val long value
- */
- public void setComponentOpt(String role, String option, long val) {
- setComponentOpt(role, option, Long.toString(val));
- }
-
- /**
- * append to a component option
- * @param key key
- * @return value
- *
- */
- public String appendComponentOpt(String role, String key, String value) {
- if (SliderUtils.isUnset(value)) {
- return null;
- }
- MapOperations roleopts = getComponent(role);
- if (roleopts == null) {
- return null;
- }
-
- if (roleopts.containsKey(key)) {
- roleopts.put(key, roleopts.get(key) + "," + value);
- } else {
- roleopts.put(key, value);
- }
- return roleopts.get(key);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/InputPropertiesValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/InputPropertiesValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/InputPropertiesValidator.java
deleted file mode 100644
index 237c240..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/InputPropertiesValidator.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.core.conf;
-
-import org.apache.slider.core.exceptions.BadConfigException;
-
-/**
- *
- */
-public interface InputPropertiesValidator {
- void validate(ConfTreeOperations props) throws BadConfigException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c82f36c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ResourcesInputPropertiesValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ResourcesInputPropertiesValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ResourcesInputPropertiesValidator.java
deleted file mode 100644
index 19f6f8d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ResourcesInputPropertiesValidator.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.core.conf;
-
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.core.exceptions.BadConfigException;
-
-/**
- *
- */
-public class ResourcesInputPropertiesValidator
- extends AbstractInputPropertiesValidator {
-
- void validatePropertyNamePrefix(String key) throws BadConfigException {
- if (!key.startsWith("yarn.") && !key.equals(ResourceKeys.UNIQUE_NAMES)) {
- throw new BadConfigException(
- "argument %s does not have 'yarn.' prefix", key);
- }
- }
-
- protected void validateGlobalProperties(ConfTreeOperations props)
- throws BadConfigException {
- for (String key : props.getGlobalOptions().keySet()) {
- validatePropertyNamePrefix(key);
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org