You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2016/05/19 19:47:30 UTC
[41/50] [abbrv] hadoop git commit: MAPREDUCE-6686. Add a way to
download the job config from the mapred CLI (rkanter)
MAPREDUCE-6686. Add a way to download the job config from the mapred CLI (rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/992a4935
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/992a4935
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/992a4935
Branch: refs/heads/HDFS-7240
Commit: 992a49353f0bdbc71534bfeeb134112c5624b94b
Parents: f04c81c
Author: Robert Kanter <rk...@apache.org>
Authored: Wed May 18 12:16:21 2016 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Wed May 18 12:16:21 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/mapreduce/tools/CLI.java | 35 ++++++++++++++++++--
.../src/site/markdown/MapredCommands.md | 3 +-
.../hadoop/mapreduce/TestMRJobClient.java | 28 ++++++++++++++++
3 files changed, 63 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/992a4935/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
index 4f55792..924af58 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java
@@ -39,6 +39,8 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.TIPStatus;
@@ -106,6 +108,7 @@ public class CLI extends Configured implements Tool {
int fromEvent = 0;
int nEvents = 0;
int jpvalue = 0;
+ String configOutFile = null;
boolean getStatus = false;
boolean getCounter = false;
boolean killJob = false;
@@ -121,6 +124,7 @@ public class CLI extends Configured implements Tool {
boolean failTask = false;
boolean setJobPriority = false;
boolean logs = false;
+ boolean downloadConfig = false;
if ("-submit".equals(cmd)) {
if (argv.length != 2) {
@@ -295,6 +299,14 @@ public class CLI extends Configured implements Tool {
displayUsage(cmd);
return exitCode;
}
+ } else if ("-config".equals(cmd)) {
+ downloadConfig = true;
+ if (argv.length != 3) {
+ displayUsage(cmd);
+ return exitCode;
+ }
+ jobid = argv[1];
+ configOutFile = argv[2];
} else {
displayUsage(cmd);
return exitCode;
@@ -473,6 +485,22 @@ public class CLI extends Configured implements Tool {
System.out.println(e.getMessage());
}
}
+ } else if (downloadConfig) {
+ Job job = getJob(JobID.forName(jobid));
+ if (job == null) {
+ System.out.println("Could not find job " + jobid);
+ } else {
+ String jobFile = job.getJobFile();
+ if (jobFile == null || jobFile.isEmpty()) {
+ System.out.println("Config file for job " + jobFile +
+ " could not be found.");
+ } else {
+ Path configPath = new Path(jobFile);
+ FileSystem fs = FileSystem.get(getConf());
+ fs.copyToLocalFile(configPath, new Path(configOutFile));
+ exitCode = 0;
+ }
+ }
}
} catch (RemoteException re) {
IOException unwrappedException = re.unwrapRemoteException();
@@ -549,7 +577,9 @@ public class CLI extends Configured implements Tool {
} else if ("-logs".equals(cmd)) {
System.err.println(prefix + "[" + cmd +
" <job-id> <task-attempt-id>]. " +
- " <task-attempt-id> is optional to get task attempt logs.");
+ " <task-attempt-id> is optional to get task attempt logs.");
+ } else if ("-config".equals(cmd)) {
+ System.err.println(prefix + "[" + cmd + " <job-id> <file>]");
} else {
System.err.printf(prefix + "<command> <args>%n");
System.err.printf("\t[-submit <job-file>]%n");
@@ -571,7 +601,8 @@ public class CLI extends Configured implements Tool {
"Valid values for <task-state> are " + taskStates);
System.err.printf("\t[-kill-task <task-attempt-id>]%n");
System.err.printf("\t[-fail-task <task-attempt-id>]%n");
- System.err.printf("\t[-logs <job-id> <task-attempt-id>]%n%n");
+ System.err.printf("\t[-logs <job-id> <task-attempt-id>]%n");
+ System.err.printf("\t[-config <job-id> <file>%n%n");
ToolRunner.printGenericCommandUsage(System.out);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/992a4935/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
index 64589d9..f312d31 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
@@ -81,7 +81,7 @@ Copy file or directories recursively. More information can be found at
Command to interact with Map Reduce Jobs.
-Usage: `mapred job | [GENERIC_OPTIONS] | [-submit <job-file>] | [-status <job-id>] | [-counter <job-id> <group-name> <counter-name>] | [-kill <job-id>] | [-events <job-id> <from-event-#> <#-of-events>] | [-history [all] <jobHistoryFile|jobId> [-outfile <file>] [-format <human|json>]] | [-list [all]] | [-kill-task <task-id>] | [-fail-task <task-id>] | [-set-priority <job-id> <priority>] | [-list-active-trackers] | [-list-blacklisted-trackers] | [-list-attempt-ids <job-id> <task-type> <task-state>] [-logs <job-id> <task-attempt-id>]`
+Usage: `mapred job | [GENERIC_OPTIONS] | [-submit <job-file>] | [-status <job-id>] | [-counter <job-id> <group-name> <counter-name>] | [-kill <job-id>] | [-events <job-id> <from-event-#> <#-of-events>] | [-history [all] <jobHistoryFile|jobId> [-outfile <file>] [-format <human|json>]] | [-list [all]] | [-kill-task <task-id>] | [-fail-task <task-id>] | [-set-priority <job-id> <priority>] | [-list-active-trackers] | [-list-blacklisted-trackers] | [-list-attempt-ids <job-id> <task-type> <task-state>] [-logs <job-id> <task-attempt-id>] [-config <job-id> <file>]`
| COMMAND\_OPTION | Description |
|:---- |:---- |
@@ -99,6 +99,7 @@ Usage: `mapred job | [GENERIC_OPTIONS] | [-submit <job-file>] | [-status <job-id
| -list-blacklisted-trackers | List the black listed task trackers in the cluster. This command is not supported in MRv2 based cluster. |
| -list-attempt-ids *job-id* *task-type* *task-state* | List the attempt-ids based on the task type and the status given. Valid values for task-type are REDUCE, MAP. Valid values for task-state are running, pending, completed, failed, killed. |
| -logs *job-id* *task-attempt-id* | Dump the container log for a job if taskAttemptId is not specified, otherwise dump the log for the task with the specified taskAttemptId. The logs will be dumped in system out. |
+| -config *job-id* *file* | Download the job configuration file. |
### `pipes`
http://git-wip-us.apache.org/repos/asf/hadoop/blob/992a4935/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
index 8fe9078..3a33bb1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMRJobClient.java
@@ -168,6 +168,8 @@ public class TestMRJobClient extends ClusterMapReduceTestCase {
testfailTask(conf);
// kill job
testKillJob(conf);
+ // download job config
+ testConfig(jobId, conf);
}
/**
@@ -535,6 +537,32 @@ public class TestMRJobClient extends ClusterMapReduceTestCase {
}
/**
+ * download job config
+ */
+ private void testConfig(String jobId, Configuration conf) throws Exception {
+ CLI jc = createJobClient();
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+
+ // bad arguments
+ int exitCode = runTool(conf, jc, new String[] { "-config" }, out);
+ assertEquals("Exit code", -1, exitCode);
+ exitCode = runTool(conf, jc, new String[] { "-config job_invalid foo.xml" },
+ out);
+ assertEquals("Exit code", -1, exitCode);
+
+ // good arguments
+ File outFile = File.createTempFile("config", ".xml");
+ exitCode = runTool(conf, jc, new String[] { "-config", jobId,
+ outFile.toString()}, out);
+ assertEquals("Exit code", 0, exitCode);
+ BufferedReader br = new BufferedReader(new FileReader(outFile));
+ String line = br.readLine();
+ br.close();
+ assertEquals("<?xml version=\"1.0\" encoding=\"UTF-8\" " +
+ "standalone=\"no\"?><configuration>", line);
+ }
+
+ /**
* print job events list
*/
private void testJobEvents(String jobId, Configuration conf) throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org