You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2011/05/27 09:53:16 UTC
svn commit: r1128184 - in /hadoop/mapreduce/branches/MR-279: ./
src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/
src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/
src/contrib/mumak/src/test/org/apache/hadoop/mapred/ src/cont...
Author: vinodkv
Date: Fri May 27 07:53:15 2011
New Revision: 1128184
URL: http://svn.apache.org/viewvc?rev=1128184&view=rev
Log:
Making streaming -file option work. Also minor fixes for successful compilation of contrib tests. (vinodkv)
Modified:
hadoop/mapreduce/branches/MR-279/CHANGES.txt
hadoop/mapreduce/branches/MR-279/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java
hadoop/mapreduce/branches/MR-279/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java
hadoop/mapreduce/branches/MR-279/src/contrib/mumak/src/test/org/apache/hadoop/mapred/MockSimulatorJobTracker.java
hadoop/mapreduce/branches/MR-279/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorJobTracker.java
hadoop/mapreduce/branches/MR-279/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTaskCommit.java
Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1128184&r1=1128183&r2=1128184&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Fri May 27 07:53:15 2011
@@ -4,6 +4,9 @@ Trunk (unreleased changes)
MAPREDUCE-279
+ Making streaming -file option work. Also minor fixes for successful
+ compilation of contrib tests. (vinodkv)
+
Fix for public dist cache to work with non default hdfs (mahadev &ddas)
Fixed an NPE during handling of unnecessary reservations in CS. (acmurthy)
Modified: hadoop/mapreduce/branches/MR-279/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java?rev=1128184&r1=1128183&r2=1128184&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/CapacityTestUtils.java Fri May 27 07:53:15 2011
@@ -668,8 +668,7 @@ public class CapacityTestUtils {
numTrackers, 0,
ttExpiryInterval, maps, reduces,
numTrackers * maxMapTasksPerTracker,
- numTrackers * maxReduceTasksPerTracker,
- JobTracker.State.RUNNING);
+ numTrackers * maxReduceTasksPerTracker);
}
public int getNumberOfUniqueHosts() {
Modified: hadoop/mapreduce/branches/MR-279/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java?rev=1128184&r1=1128183&r2=1128184&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/contrib/fairscheduler/src/test/org/apache/hadoop/mapred/TestFairScheduler.java Fri May 27 07:53:15 2011
@@ -362,8 +362,7 @@ public class TestFairScheduler extends T
return new ClusterStatus(numTrackers, 0,
ttExpiryInterval, maps, reduces,
numTrackers * maxMapTasksPerTracker,
- numTrackers * maxReduceTasksPerTracker,
- JobTracker.State.RUNNING);
+ numTrackers * maxReduceTasksPerTracker);
}
@Override
Modified: hadoop/mapreduce/branches/MR-279/src/contrib/mumak/src/test/org/apache/hadoop/mapred/MockSimulatorJobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/contrib/mumak/src/test/org/apache/hadoop/mapred/MockSimulatorJobTracker.java?rev=1128184&r1=1128183&r2=1128184&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/contrib/mumak/src/test/org/apache/hadoop/mapred/MockSimulatorJobTracker.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/contrib/mumak/src/test/org/apache/hadoop/mapred/MockSimulatorJobTracker.java Fri May 27 07:53:15 2011
@@ -27,6 +27,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.ProtocolSignature;
import org.apache.hadoop.mapred.TaskStatus.State;
import org.apache.hadoop.mapred.TaskStatus.Phase;
import org.apache.hadoop.mapreduce.ClusterMetrics;
@@ -379,12 +380,6 @@ public class MockSimulatorJobTracker imp
}
@Override
- public org.apache.hadoop.mapreduce.server.jobtracker.State getJobTrackerState()
- throws IOException, InterruptedException {
- throw new UnsupportedOperationException();
- }
-
- @Override
public QueueInfo getQueue(String queueName) throws IOException,
InterruptedException {
throw new UnsupportedOperationException();
@@ -467,4 +462,11 @@ public class MockSimulatorJobTracker imp
) throws IOException,InterruptedException{
return 0;
}
+
+ @Override
+ public ProtocolSignature getProtocolSignature(String protocol,
+ long clientVersion, int clientMethodsHash) throws IOException {
+ return ProtocolSignature.getProtocolSigature(
+ this, protocol, clientVersion, clientMethodsHash);
+ }
}
Modified: hadoop/mapreduce/branches/MR-279/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorJobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorJobTracker.java?rev=1128184&r1=1128183&r2=1128184&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorJobTracker.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorJobTracker.java Fri May 27 07:53:15 2011
@@ -145,7 +145,8 @@ public class TestSimulatorJobTracker {
// HashSet<> numLaunchTaskActions
for (TaskTrackerAction action : actions) {
if (action instanceof SimulatorLaunchTaskAction) {
- Task task = ((SimulatorLaunchTaskAction) action).getTask();
+ TTTask ttTask = ((SimulatorLaunchTaskAction) action).getTask();
+ Task task = ttTask.getTask();
numLaunchTaskActions++;
TaskAttemptID taskId = task.getTaskID();
Modified: hadoop/mapreduce/branches/MR-279/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java?rev=1128184&r1=1128183&r2=1128184&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/contrib/streaming/src/java/org/apache/hadoop/streaming/StreamJob.java Fri May 27 07:53:15 2011
@@ -22,8 +22,10 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
@@ -43,6 +45,7 @@ import org.apache.hadoop.mapreduce.MRCon
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.mapred.FileInputFormat;
@@ -276,19 +279,25 @@ public class StreamJob implements Tool {
if (values != null && values.length > 0) {
LOG.warn("-file option is deprecated, please use generic option" +
" -files instead.");
- StringBuilder unpackRegex = new StringBuilder(
- config_.getPattern(MRJobConfig.JAR_UNPACK_PATTERN,
- JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern());
+
+ String fileList = null;
for (String file : values) {
packageFiles_.add(file);
- String fname = new File(file).getName();
- unpackRegex.append("|(?:").append(Pattern.quote(fname)).append(")");
+ try {
+ URI pathURI = new URI(file);
+ Path path = new Path(pathURI);
+ FileSystem localFs = FileSystem.getLocal(config_);
+ String finalPath = path.makeQualified(localFs).toString();
+ fileList = fileList == null ? finalPath : fileList + "," + finalPath;
+ } catch (Exception e) {
+ throw new IllegalArgumentException(e);
+ }
}
- config_.setPattern(MRJobConfig.JAR_UNPACK_PATTERN,
- Pattern.compile(unpackRegex.toString()));
+ config_.set("tmpfiles", config_.get("tmpfiles", "") +
+ (fileList == null ? "" : fileList));
validate(packageFiles_);
}
-
+
String fsName = cmdLine.getOptionValue("dfs");
if (null != fsName){
LOG.warn("-dfs option is deprecated, please use -fs instead.");
Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTaskCommit.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTaskCommit.java?rev=1128184&r1=1128183&r2=1128184&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTaskCommit.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/TestTaskCommit.java Fri May 27 07:53:15 2011
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.ProtocolSignature;
import org.apache.hadoop.mapred.SortedRanges.Range;
import org.apache.hadoop.mapreduce.TaskType;
@@ -168,6 +169,13 @@ public class TestTaskCommit extends Hado
long[] sizes) throws IOException {
// NOTHING
}
+
+ @Override
+ public ProtocolSignature getProtocolSignature(String protocol,
+ long clientVersion, int clientMethodsHash) throws IOException {
+ return ProtocolSignature.getProtocolSigature(
+ this, protocol, clientVersion, clientMethodsHash);
+ }
}
/**