You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/10/12 02:58:22 UTC
[34/50] git commit: SLIDER-365 slider "resolve" command to retrieve
service record
SLIDER-365 slider "resolve" command to retrieve service record
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bfaea962
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bfaea962
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bfaea962
Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: bfaea9629ed7aa11e6c882e4a7533cc081cfc9dc
Parents: fa53ef9
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 9 11:29:10 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 9 11:32:26 2014 -0700
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 57 +++++-------
.../slider/common/params/ActionResolveArgs.java | 95 ++++++++++++++++++--
.../TestStandaloneYarnRegistryAM.groovy | 64 ++++++++-----
.../apache/slider/test/SliderTestUtils.groovy | 21 ++++-
4 files changed, 171 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfaea962/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 7daa2b8..2944e2a 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.alias.CredentialProvider;
@@ -134,13 +135,11 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileNotFoundException;
-import java.io.FileWriter;
import java.io.IOException;
import java.io.StringWriter;
import java.io.Writer;
import java.net.InetSocketAddress;
import java.net.URISyntaxException;
-import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -149,7 +148,6 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Properties;
import java.util.Set;
import java.util.regex.Pattern;
@@ -394,14 +392,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
YarnAppListClient =
new YarnAppListClient(yarnClient, getUsername(), getConfig());
// create the filesystem
- sliderFileSystem = new SliderFileSystem(getConfig());
-
- // and the registry
-/*
- YARNRegistryClient =
- new YARNRegistryClient(yarnClient, getUsername(), getConfig());
-*/
-
+ sliderFileSystem = new SliderFileSystem(getConfig());
}
/**
@@ -2211,40 +2202,38 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
args.validate();
RegistryOperations operations = getRegistryOperations();
String path = args.path;
- Collection<ServiceRecord> serviceRecords;
ServiceRecordMarshal serviceRecordMarshal = new ServiceRecordMarshal();
- File outputPath = args.out;
try {
if (args.list) {
+ File destDir = args.destdir;
+
Map<String, ServiceRecord> recordMap =
listServiceRecords(operations, path);
- serviceRecords = recordMap.values();
- // list records out
- StringBuilder builder = new StringBuilder(1024);
- for (Entry<String, ServiceRecord> recordEntry : recordMap
- .entrySet()) {
+
+ for (Entry<String, ServiceRecord> recordEntry : recordMap.entrySet()) {
+ String name = recordEntry.getKey();
ServiceRecord instance = recordEntry.getValue();
- builder.append("\"").append(recordEntry.getKey()).append("\":\n");
- builder.append(serviceRecordMarshal.toJson(instance));
- builder.append("}\n");
- }
- String records = builder.toString();
- if (outputPath == null) {
- print(records);
- } else {
- SliderUtils.write(outputPath, records.getBytes("UTF-8"), false);
+ String json = serviceRecordMarshal.toJson(instance);
+ if (destDir == null) {
+ print(name);
+ print(json);
+ } else {
+ String filename = RegistryPathUtils.lastPathEntry(name) + ".json";
+ File jsonFile = new File(destDir, filename);
+ SliderUtils.write(jsonFile,
+ serviceRecordMarshal.toBytes(instance), true);
+ }
}
} else {
// resolve single entry
ServiceRecord instance = resolve(path);
- serviceRecords = new ArrayList<ServiceRecord>(1);
- serviceRecords.add(instance);
- // write out JSON content
- if (outputPath != null) {
- byte[] data = serviceRecordMarshal.toBytes(instance);
- SliderUtils.write(outputPath, data, false);
+ File outFile = args.out;
+ if (args.destdir != null) {
+ outFile = new File(args.destdir, RegistryPathUtils.lastPathEntry(path));
+ }
+ if (outFile != null) {
+ SliderUtils.write(outFile, serviceRecordMarshal.toBytes(instance), true);
} else {
- // print to the console
print(serviceRecordMarshal.toJson(instance));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfaea962/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
index cf5611c..944be1b 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
@@ -20,7 +20,7 @@ package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
-import org.apache.slider.common.SliderKeys;
+import org.apache.commons.lang.StringUtils;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.UsageException;
@@ -45,10 +45,11 @@ public class ActionResolveArgs extends AbstractActionArgs {
public static final String USAGE =
"Usage: " + SliderActions.ACTION_RESOLVE
+ " "
- + Arguments.ARG_PATH + " <path> "
- + "[" + Arguments.ARG_LIST + "] "
- + "[" + Arguments.ARG_VERBOSE + "] "
- + "[" + Arguments.ARG_OUTPUT + " <filename> ] "
+ + ARG_PATH + " <path> "
+ + "[" + ARG_LIST + "] "
+ + "[" + ARG_VERBOSE + "] "
+ + "[" + ARG_OUTPUT + " <filename> ] "
+ + "[" + ARG_DESTDIR + " <directory> ] "
;
public ActionResolveArgs() {
}
@@ -71,17 +72,95 @@ public class ActionResolveArgs extends AbstractActionArgs {
description = "list services")
public boolean list;
-
@Parameter(names = {ARG_PATH},
description = "resolve a path")
public String path;
+ @Parameter(names = {ARG_DESTDIR},
+ description = "destination directory for operations")
+ public File destdir;
+
@Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "Output destination")
+ description = "dest file")
public File out;
@Parameter(names = {ARG_VERBOSE},
description = "verbose output")
public boolean verbose;
-
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder(ACTION_RESOLVE).append(" ");
+ sb.append(ARG_PATH).append(" ").append(path).append(" ");
+ if (list) {
+ sb.append(ARG_LIST).append(" ");
+ }
+ if (destdir != null) {
+ sb.append(ARG_DESTDIR).append(" ").append(destdir).append(" ");
+ }
+ if (out != null) {
+ sb.append(ARG_OUTPUT).append(" ").append(out).append(" ");
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public void validate() throws BadCommandArgumentsException, UsageException {
+ super.validate();
+ if (StringUtils.isEmpty(path)) {
+ throw new BadCommandArgumentsException("Missing mandatory argument "
+ + ARG_PATH);
+ }
+ if (list && out != null) {
+ throw new BadCommandArgumentsException("Argument "
+ + ARG_OUTPUT +
+ " not supported for " + ARG_LIST);
+ }
+ if (out != null && destdir != null) {
+ throw new BadCommandArgumentsException(
+ ARG_OUTPUT + " and " + ARG_DESTDIR + " cannot be used together"
+ );
+ }
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ public void setPath(String path) {
+ this.path = path;
+ }
+
+ public boolean isList() {
+ return list;
+ }
+
+ public void setList(boolean list) {
+ this.list = list;
+ }
+
+ public File getDestdir() {
+ return destdir;
+ }
+
+ public void setDestdir(File destdir) {
+ this.destdir = destdir;
+ }
+
+ public File getOut() {
+ return out;
+ }
+
+ public void setOut(File out) {
+ this.out = out;
+ }
+
+ public boolean isVerbose() {
+ return verbose;
+ }
+
+ public void setVerbose(boolean verbose) {
+ this.verbose = verbose;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfaea962/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index 84dcc50..ca200fd 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -20,6 +20,8 @@ package org.apache.slider.agent.standalone
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.commons.io.IOUtils
+import org.apache.hadoop.fs.FileUtil
import org.apache.hadoop.fs.PathNotFoundException
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.api.records.YarnApplicationState
@@ -31,7 +33,10 @@ import org.apache.hadoop.registry.client.types.RegistryPathStatus
import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes
import org.apache.slider.common.params.ActionResolveArgs
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.exceptions.BadCommandArgumentsException
import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
+import org.apache.slider.core.main.LauncherExitCodes
import static org.apache.hadoop.registry.client.binding.RegistryUtils.*
import org.apache.slider.agent.AgentMiniClusterTestBase
@@ -174,37 +179,52 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
assert null != serviceRecord.getInternalEndpoint(AGENT_SECURE_REST_API)
// use the resolve operation
+
describe "resolve CLI action"
- ActionResolveArgs resolveArgs = new ActionResolveArgs()
- resolveArgs.path = recordsPath;
- resolveArgs.list = true;
+ File destDir= new File("target/resolve")
+ ServiceRecordMarshal serviceRecordMarshal = new ServiceRecordMarshal()
+ FileUtil.fullyDelete(destDir)
+ File resolveListDir= new File(destDir, "list")
+ ActionResolveArgs resolveList = new ActionResolveArgs(
+ path:recordsPath,
+ list:true,
+ verbose:true)
+
// to stdout
- client.actionResolve(resolveArgs)
+ client.actionResolve(resolveList)
// to a file
- File destFile = new File("target/resolve.json")
- destFile.delete()
- resolveArgs.out = destFile
- client.actionResolve(resolveArgs)
- assert destFile.exists()
- destFile.delete()
+ resolveList.out = resolveListDir;
+ try {
+ client.actionResolve(resolveList)
+ } catch (BadCommandArgumentsException ex) {
+ assertExceptionDetails(ex, LauncherExitCodes.EXIT_COMMAND_ARGUMENT_ERROR,
+ Arguments.ARG_OUTPUT)
+ }
+ resolveList.out = null
+ resolveList.destdir = resolveListDir
+ client.actionResolve(resolveList)
+ File resolvedFile = new File(resolveListDir, clustername+".json")
+ assertFileExists("resolved file", resolvedFile)
+ def recordFromList = serviceRecordMarshal.fromFile(resolvedFile)
// look at a single record
- resolveArgs.out = null;
- resolveArgs.list = false;
- resolveArgs.path = recordsPath +"/"+ clustername
+ ActionResolveArgs resolveRecordCommand = new ActionResolveArgs(
+ path: recordsPath + "/" + clustername)
// to stdout
- client.actionResolve(resolveArgs)
- resolveArgs.out = destFile
- client.actionResolve(resolveArgs)
- assert destFile.exists()
- ServiceRecordMarshal serviceRecordMarshal = new ServiceRecordMarshal()
- def recordFromFile = serviceRecordMarshal.fromFile(destFile)
+ client.actionResolve(resolveRecordCommand)
+
+ // then to a file
+ FileUtil.fullyDelete(destDir)
+ File singleFile = new File(destDir, "singlefile.json")
+ singleFile.delete()
+ resolveRecordCommand.out = singleFile
+ client.actionResolve(resolveRecordCommand)
+ assertFileExists("\"slider $resolveRecordCommand\"", singleFile)
+ def recordFromFile = serviceRecordMarshal.fromFile(singleFile)
assert recordFromFile[YarnRegistryAttributes.YARN_ID] ==
serviceRecord[YarnRegistryAttributes.YARN_ID]
assert recordFromFile[YarnRegistryAttributes.YARN_PERSISTENCE] ==
serviceRecord[YarnRegistryAttributes.YARN_PERSISTENCE]
-
-
// hit the registry web page
def registryEndpoint = serviceRecord.getExternalEndpoint(
@@ -402,7 +422,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
// create a new registry args with the defaults back in
registryArgs = new ActionRegistryArgs(clustername)
registryArgs.getConf = yarn_site_config
- registryArgs.dest = outputDir
+ registryArgs.out = outputDir
describe registryArgs.toString()
client.actionRegistry(registryArgs)
assert new File(outputDir, yarn_site_config + ".xml").exists()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfaea962/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 0b45a5d..d09d280 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -79,13 +79,13 @@ class SliderTestUtils extends Assert {
}
public static void skip(String message) {
- log.warn("Skipping test: " + message)
+ log.warn("Skipping test: {}", message)
Assume.assumeTrue(message, false);
}
public static void assume(boolean condition, String message) {
if (!condition) {
- log.warn("Skipping test: " + message)
+ log.warn("Skipping test: {}", message)
Assume.assumeTrue(message, false);
}
}
@@ -793,4 +793,21 @@ class SliderTestUtils extends Assert {
public String toURIArg(File file) {
file.absoluteFile.toURI().toString()
}
+
+ /**
+ * Assert a file exists; fails with a listing of the parent dir
+ * @param text text for front of message
+ * @param file file to look for
+ * @throws FileNotFoundException
+ */
+ public void assertFileExists(String text, File file) {
+ if (!file.exists()) {
+ def parent = file.parentFile
+ def files = parent.list()
+ StringBuilder builder = new StringBuilder()
+ builder.append("${parent.absolutePath}:\n")
+ files.each { String name-> builder.append(" $name\n")}
+ throw new FileNotFoundException("$text: file $file not found in $builder")
+ }
+ }
}