You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by at...@apache.org on 2011/12/29 19:25:50 UTC
svn commit: r1225612 - in /hadoop/common/branches/HDFS-1623: ./
hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/
hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/
Author: atm
Date: Thu Dec 29 18:25:48 2011
New Revision: 1225612
URL: http://svn.apache.org/viewvc?rev=1225612&view=rev
Log:
Merge trunk into HA branch.
Added:
hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMRFramework.java
- copied unchanged from r1225610, hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMRFramework.java
Modified:
hadoop/common/branches/HDFS-1623/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamUtil.java
hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamJob.java
Propchange: hadoop/common/branches/HDFS-1623/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Dec 29 18:25:48 2011
@@ -1 +1 @@
-/hadoop/common/trunk:1152502-1225257
+/hadoop/common/trunk:1152502-1225610
Modified: hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java?rev=1225612&r1=1225611&r2=1225612&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java Thu Dec 29 18:25:48 2011
@@ -255,6 +255,13 @@ public class StreamJob implements Tool {
}
if (cmdLine != null) {
+ @SuppressWarnings("unchecked")
+ List<String> args = cmdLine.getArgList();
+ if(args != null && args.size() > 0) {
+ fail("Found " + args.size() + " unexpected arguments on the " +
+ "command line " + args);
+ }
+
detailedUsage_ = cmdLine.hasOption("info");
if (cmdLine.hasOption("help") || detailedUsage_) {
printUsage = true;
Modified: hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamUtil.java?rev=1225612&r1=1225611&r2=1225612&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamUtil.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamUtil.java Thu Dec 29 18:25:48 2011
@@ -18,16 +18,19 @@
package org.apache.hadoop.streaming;
-import java.io.*;
-import java.net.*;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.URL;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
+import org.apache.hadoop.mapreduce.MRConfig;
/**
* Utilities used in streaming
@@ -189,6 +192,8 @@ public class StreamUtil {
}
public static boolean isLocalJobTracker(JobConf job) {
- return job.get(JTConfig.JT_IPC_ADDRESS, "local").equals("local");
+ String framework =
+ job.get(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
+ return framework.equals(MRConfig.LOCAL_FRAMEWORK_NAME);
}
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamJob.java?rev=1225612&r1=1225611&r2=1225612&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamJob.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamJob.java Thu Dec 29 18:25:48 2011
@@ -32,7 +32,18 @@ import static org.junit.Assert.*;
* This class tests hadoop Streaming's StreamJob class.
*/
public class TestStreamJob {
-
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testCreateJobWithExtraArgs() throws IOException {
+ ArrayList<String> dummyArgs = new ArrayList<String>();
+ dummyArgs.add("-input"); dummyArgs.add("dummy");
+ dummyArgs.add("-output"); dummyArgs.add("dummy");
+ dummyArgs.add("-mapper"); dummyArgs.add("dummy");
+ dummyArgs.add("dummy");
+ dummyArgs.add("-reducer"); dummyArgs.add("dummy");
+ StreamJob.createJob(dummyArgs.toArray(new String[] {}));
+ }
+
@Test
public void testCreateJob() throws IOException {
JobConf job;