You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by bi...@apache.org on 2016/08/03 21:19:50 UTC
incubator-slider git commit: SLIDER-1107 address suggestions from gour
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-1107_AM_config_generation 41de54d76 -> d3923d889
SLIDER-1107 address suggestions from gour
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d3923d88
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d3923d88
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d3923d88
Branch: refs/heads/feature/SLIDER-1107_AM_config_generation
Commit: d3923d88915997f62b35ec43bbe86b232c9bcccf
Parents: 41de54d
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed Aug 3 14:16:09 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Wed Aug 3 14:16:09 2016 -0700
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 5 ++-
.../slider/common/params/SliderActions.java | 2 +-
.../slider/common/tools/CoreFileSystem.java | 31 ++++++++++----
.../slider/core/conf/ConfTreeOperations.java | 14 ------
.../slider/core/launch/AbstractLauncher.java | 10 ++---
.../providers/agent/AgentClientProvider.java | 45 +++++++++++++-------
.../providers/agent/AgentProviderService.java | 26 +++++------
.../agent/TestAgentProviderService.java | 4 ++
8 files changed, 80 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d3923d88/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index eb0630d..af134b3 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1112,7 +1112,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
@Override
public int actionResource(ActionResourceArgs resourceInfo)
throws YarnException, IOException {
- if (resourceInfo.install) {
+ if (resourceInfo.help) {
+ actionHelp(ACTION_RESOURCE);
+ return EXIT_SUCCESS;
+ } else if (resourceInfo.install) {
return actionInstallResource(resourceInfo);
} else if (resourceInfo.delete) {
return actionDeleteResource(resourceInfo);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d3923d88/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
index e209cdc..204ad9a 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
@@ -107,7 +107,7 @@ public interface SliderActions {
" Deprecated, use '" + ACTION_KEYTAB + " " + ClientArgs.ARG_INSTALL + "'.";
String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory";
String DESCRIBE_ACTION_DIAGNOSTIC = "Diagnose the configuration of the running slider application and slider client";
- String DESCRIBE_ACTION_RESOURCE = "Manage a file (install, delete, list) in the sub-folder 'resources' of the user's Slider base directory";
+ String DESCRIBE_ACTION_RESOURCE = "Manage a file (install, delete, list) in the 'resources' sub-folder of the user's Slider base directory";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d3923d88/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index 0b0f1bc..aa5edf1 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -830,23 +830,36 @@ public class CoreFileSystem {
}
public void touch(Path path, boolean overwrite) throws IOException {
- FSDataOutputStream out = fileSystem.create(path, overwrite);
- out.close();
+ FSDataOutputStream out = null;
+ try {
+ out = fileSystem.create(path, overwrite);
+ } finally {
+ IOUtils.closeStream(out);
+ }
}
public void cat(Path path, boolean overwrite, String data) throws IOException {
- FSDataOutputStream out = fileSystem.create(path, overwrite);
- byte[] bytes = data.getBytes(Charset.forName("UTF-8"));
- out.write(bytes);
- out.close();
+ FSDataOutputStream out = null;
+ try {
+ out = fileSystem.create(path, overwrite);
+ byte[] bytes = data.getBytes(Charset.forName("UTF-8"));
+ out.write(bytes);
+ } finally {
+ IOUtils.closeStream(out);
+ }
}
public String cat(Path path) throws IOException {
FileStatus status = fileSystem.getFileStatus(path);
byte[] b = new byte[(int) status.getLen()];
- FSDataInputStream in = fileSystem.open(path);
- int count = in.read(b);
- return new String(b, 0, count, UTF_8);
+ FSDataInputStream in = null;
+ try {
+ in = fileSystem.open(path);
+ int count = in.read(b);
+ return new String(b, 0, count, UTF_8);
+ } finally {
+ IOUtils.closeStream(in);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d3923d88/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
index d376c72..d24a158 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
@@ -347,20 +347,6 @@ public class ConfTreeOperations {
}
/**
- * Load from a json String. The inner conf tree is the loaded data -unresolved
- * @param json json string
- * @return loaded value
- * @throws IOException load failure
- */
- public static ConfTreeOperations fromString(String json) throws
- IOException {
- ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser();
- ConfTreeOperations ops = new ConfTreeOperations(
- confTreeSerDeser.fromJson(json) );
- return ops;
- }
-
- /**
* Build from an existing instance -which is cloned via JSON ser/deser
* @param instance the source instance
* @return loaded value
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d3923d88/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
index efc58b4..5a3eb3d 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -133,13 +133,13 @@ public abstract class AbstractLauncher extends Configured {
return localResources;
}
- public void addLocalResource(String subpath, LocalResource resource) {
- localResources.put(subpath, resource);
+ public void addLocalResource(String subPath, LocalResource resource) {
+ localResources.put(subPath, resource);
}
- public void addLocalResource(String subpath, LocalResource resource, String mountPath) {
- localResources.put(subpath, resource);
- mountPaths.put(subpath, mountPath);
+ public void addLocalResource(String subPath, LocalResource resource, String mountPath) {
+ localResources.put(subPath, resource);
+ mountPaths.put(subPath, mountPath);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d3923d88/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 3eef0b0..4c6a50b 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -27,6 +27,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.registry.client.binding.RegistryUtils;
import org.apache.slider.api.InternalKeys;
@@ -335,9 +336,21 @@ public class AgentClientProvider extends AbstractClientProvider
extractFile(zipInputStream, filePath);
if ("metainfo.xml".equals(zipEntry.getName())) {
- metaInfo = new MetainfoParser().fromXmlStream(new FileInputStream(filePath));
+ FileInputStream input = null;
+ try {
+ input = new FileInputStream(filePath);
+ metaInfo = new MetainfoParser().fromXmlStream(input);
+ } finally {
+ IOUtils.closeStream(input);
+ }
} else if ("metainfo.json".equals(zipEntry.getName())) {
- metaInfo = new MetainfoParser().fromJsonStream(new FileInputStream(filePath));
+ FileInputStream input = null;
+ try {
+ input = new FileInputStream(filePath);
+ metaInfo = new MetainfoParser().fromJsonStream(input);
+ } finally {
+ IOUtils.closeStream(input);
+ }
} else if ("clientInstallConfig-default.json".equals(zipEntry.getName())) {
try {
defaultConfig = new JSONObject(FileUtils.readFileToString(new File(filePath), Charset.defaultCharset()));
@@ -362,22 +375,22 @@ public class AgentClientProvider extends AbstractClientProvider
throw new BadConfigException(E_COULD_NOT_READ_METAINFO);
}
- String client_script = null;
+ String clientScript = null;
String clientComponent = null;
for (Component component : metaInfo.getApplication().getComponents()) {
if (component.getCategory().equals("CLIENT")) {
clientComponent = component.getName();
if (component.getCommandScript() != null) {
- client_script = component.getCommandScript().getScript();
+ clientScript = component.getCommandScript().getScript();
}
break;
}
}
- if (SliderUtils.isUnset(client_script)) {
+ if (SliderUtils.isUnset(clientScript)) {
log.info("Installing CLIENT without script");
List<Package> packages = metaInfo.getApplication().getPackages();
- if (packages != null && packages.size() > 0) {
+ if (packages.size() > 0) {
// retrieve package resources from HDFS and extract
for (Package pkg : packages) {
Path pkgPath = fileSystem.buildResourcePath(pkg.getName());
@@ -416,20 +429,22 @@ public class AgentClientProvider extends AbstractClientProvider
}
File confInstallDir;
String clientRoot = null;
- if (defaultConfig != null) {
+ if (config != null) {
try {
- clientRoot = defaultConfig.getJSONObject("global")
+ clientRoot = config.getJSONObject("global")
.getString(AgentKeys.APP_CLIENT_ROOT);
} catch (JSONException e) {
- e.printStackTrace();
+ log.info("Couldn't read {} from provided client config, falling " +
+ "back on default", AgentKeys.APP_CLIENT_ROOT);
}
}
- if (config != null) {
+ if (clientRoot == null && defaultConfig != null) {
try {
- clientRoot = config.getJSONObject("global")
+ clientRoot = defaultConfig.getJSONObject("global")
.getString(AgentKeys.APP_CLIENT_ROOT);
} catch (JSONException e) {
- e.printStackTrace();
+ log.info("Couldn't read {} from default client config, using {}",
+ AgentKeys.APP_CLIENT_ROOT, clientInstallPath);
}
}
if (clientRoot == null) {
@@ -446,7 +461,7 @@ public class AgentClientProvider extends AbstractClientProvider
confInstallDir);
}
} else {
- log.info("Installing CLIENT using script {}", client_script);
+ log.info("Installing CLIENT using script {}", clientScript);
expandAgentTar(agentPkgDir);
JSONObject commandJson = getCommandJson(defaultConfig, config, metaInfo, clientInstallPath, name);
@@ -455,13 +470,13 @@ public class AgentClientProvider extends AbstractClientProvider
file.write(commandJson.toString());
} catch (IOException e) {
- e.printStackTrace();
+ log.error("Couldn't write command.json to file");
} finally {
file.flush();
file.close();
}
- runCommand(appPkgDir, agentPkgDir, cmdDir, client_script);
+ runCommand(appPkgDir, agentPkgDir, cmdDir, clientScript);
}
} catch (IOException ioex) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d3923d88/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index bc04220..4ffae7c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.registry.client.types.Endpoint;
import org.apache.hadoop.registry.client.types.ProtocolTypes;
@@ -170,7 +171,6 @@ public class AgentProviderService extends AbstractProviderService implements
private AgentClientProvider clientProvider;
private AtomicInteger taskId = new AtomicInteger(0);
private volatile Metainfo metaInfo = null;
- private AggregateConf instanceDefinition = null;
private SliderFileSystem fileSystem = null;
private Map<String, DefaultConfig> defaultConfigs = null;
private ComponentCommandOrder commandOrder = null;
@@ -284,7 +284,6 @@ public class AgentProviderService extends AbstractProviderService implements
if (metaInfo == null) {
synchronized (syncLock) {
if (metaInfo == null) {
- this.instanceDefinition = instanceDefinition;
this.fileSystem = fileSystem;
readAndSetHeartbeatMonitoringInterval(instanceDefinition);
initializeAgentDebugCommands(instanceDefinition);
@@ -702,12 +701,15 @@ public class AgentProviderService extends AbstractProviderService implements
}
Path destPath = new Path(parentDir, resource.getName());
if (!fileSystem.getFileSystem().exists(destPath)) {
- FSDataOutputStream os = fileSystem.getFileSystem().create(destPath);
- byte[] contents = FileUtils.readFileToByteArray(resource);
- os.write(contents, 0, contents.length);
-
- os.flush();
- os.close();
+ FSDataOutputStream os = null;
+ try {
+ os = fileSystem.getFileSystem().create(destPath);
+ byte[] contents = FileUtils.readFileToByteArray(resource);
+ os.write(contents, 0, contents.length);
+ os.flush();
+ } finally {
+ IOUtils.closeStream(os);
+ }
log.info("Uploaded {} to localization path {}", resource, destPath);
} else {
log.info("Resource {} already existed at localization path {}", resource,
@@ -1362,7 +1364,7 @@ public class AgentProviderService extends AbstractProviderService implements
// identify client component
Component client = null;
for (Component component : getMetaInfo().getApplication().getComponents()) {
- if (component != null && component.getCategory().equals("CLIENT")) {
+ if (component.getCategory().equals("CLIENT")) {
client = component;
break;
}
@@ -1373,7 +1375,7 @@ public class AgentProviderService extends AbstractProviderService implements
}
// register AM-generated client configs
- ConfTreeOperations appConf = instanceDefinition.getAppConfOperations();
+ ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
MapOperations clientOperations = appConf.getOrAddComponent(client.getName());
appConf.resolve();
if (!clientOperations.getOptionBool(AgentKeys.AM_CONFIG_GENERATION,
@@ -1765,7 +1767,7 @@ public class AgentProviderService extends AbstractProviderService implements
Application application = getMetaInfo().getApplication();
if ((!canAnyMasterPublishConfig() || canPublishConfig(componentGroup)) &&
- !instanceDefinition.getAppConfOperations().getComponentOptBool(
+ !getAmState().getAppConfSnapshot().getComponentOptBool(
componentGroup, AgentKeys.AM_CONFIG_GENERATION, false)) {
// If no Master can explicitly publish then publish if its a master
// Otherwise, wait till the master that can publish is ready
@@ -1890,7 +1892,7 @@ public class AgentProviderService extends AbstractProviderService implements
simpleEntries.put(entry.getKey(), entry.getValue().get(0).getValue());
}
}
- if (!instanceDefinition.getAppConfOperations().getComponentOptBool(
+ if (!getAmState().getAppConfSnapshot().getComponentOptBool(
groupName, AgentKeys.AM_CONFIG_GENERATION, false)) {
publishApplicationInstanceData(groupName, groupName,
simpleEntries.entrySet());
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d3923d88/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index af1c61f..0d56ff7 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -902,6 +902,8 @@ public class TestAgentProviderService {
doReturn(access).when(mockAps).getAmState();
PublishedExportsSet pubExpSet = new PublishedExportsSet();
expect(access.getPublishedExportsSet()).andReturn(pubExpSet).anyTimes();
+ expect(access.getAppConfSnapshot()).andReturn(new ConfTreeOperations(
+ new ConfTree())).anyTimes();
replay(access);
Map<String, String> ports = new HashMap<String, String>();
@@ -1016,6 +1018,8 @@ public class TestAgentProviderService {
doReturn(access).when(mockAps).getAmState();
PublishedExportsSet pubExpSet = new PublishedExportsSet();
expect(access.getPublishedExportsSet()).andReturn(pubExpSet).anyTimes();
+ expect(access.getAppConfSnapshot()).andReturn(new ConfTreeOperations(
+ new ConfTree())).anyTimes();
replay(access);
mockAps.publishConfigAndExportGroups(hb, componentStatus, "HBASE_MASTER");