You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by at...@apache.org on 2013/04/13 01:05:33 UTC
svn commit: r1467511 - in /hadoop/common/branches/HDFS-347: ./
hadoop-client/ hadoop-tools/hadoop-gridmix/
hadoop-tools/hadoop-gridmix/dev-support/
hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/
hadoop-tools/hadoop-gridmix/...
Author: atm
Date: Fri Apr 12 23:05:28 2013
New Revision: 1467511
URL: http://svn.apache.org/r1467511
Log:
Merge trunk into the HDFS-347 branch.
Added:
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/dev-support/
- copied from r1467500, hadoop/common/trunk/hadoop-tools/hadoop-gridmix/dev-support/
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml
- copied unchanged from r1467500, hadoop/common/trunk/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/CommonJobTest.java
- copied unchanged from r1467500, hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/CommonJobTest.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestDistCacheEmulation.java
- copied unchanged from r1467500, hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestDistCacheEmulation.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridMixClasses.java
- copied unchanged from r1467500, hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridMixClasses.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSubmission.java
- copied unchanged from r1467500, hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSubmission.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestLoadJob.java
- copied unchanged from r1467500, hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestLoadJob.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestSleepJob.java
- copied unchanged from r1467500, hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestSleepJob.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/resources/data/wordcount.json
- copied unchanged from r1467500, hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/resources/data/wordcount.json
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/resources/data/wordcount2.json
- copied unchanged from r1467500, hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/resources/data/wordcount2.json
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/bin/
- copied from r1467500, hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/bin/
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/bin/cat.cmd
- copied unchanged from r1467500, hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/bin/cat.cmd
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/bin/xargs_cat.cmd
- copied unchanged from r1467500, hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/bin/xargs_cat.cmd
Modified:
hadoop/common/branches/HDFS-347/ (props changed)
hadoop/common/branches/HDFS-347/hadoop-client/pom.xml
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/pom.xml
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobMonitor.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SerialJobFactory.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SleepJob.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobFactory.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobProducer.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/GridmixTestUtils.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/pom.xml
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreaming.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingKeyValue.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java
hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java
Propchange: hadoop/common/branches/HDFS-347/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-2.0.4-alpha:r1463804
Merged /hadoop/common/trunk:r1462626-1467500
Modified: hadoop/common/branches/HDFS-347/hadoop-client/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-client/pom.xml?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-client/pom.xml (original)
+++ hadoop/common/branches/HDFS-347/hadoop-client/pom.xml Fri Apr 12 23:05:28 2013
@@ -144,10 +144,6 @@
<artifactId>jetty</artifactId>
</exclusion>
<exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </exclusion>
- <exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
Modified: hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/pom.xml?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/pom.xml (original)
+++ hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/pom.xml Fri Apr 12 23:05:28 2013
@@ -91,10 +91,25 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <findbugsXmlOutput>true</findbugsXmlOutput>
+ <xmlOutput>true</xmlOutput>
+ <excludeFilterFile>${basedir}/dev-support/findbugs-exclude.xml</excludeFilterFile>
+ <effort>Max</effort>
+ </configuration>
+ </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
@@ -116,6 +131,15 @@
</executions>
</plugin>
<plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>src/test/resources/data/*</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<configuration>
Modified: hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java Fri Apr 12 23:05:28 2013
@@ -36,7 +36,9 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapred.gridmix.GenerateData.DataStatistics;
import org.apache.hadoop.mapred.gridmix.Statistics.JobStats;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
@@ -693,7 +695,7 @@ public class Gridmix extends Configured
try {
res = ToolRunner.run(new Configuration(), new Gridmix(argv), argv);
} finally {
- System.exit(res);
+ ExitUtil.terminate(res);
}
}
@@ -800,6 +802,10 @@ public class Gridmix extends Configured
*/
void abort();
}
-
+ // it is need for tests
+ protected Summarizer getSummarizer() {
+ return summarizer;
+ }
+
}
Modified: hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobMonitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobMonitor.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobMonitor.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobMonitor.java Fri Apr 12 23:05:28 2013
@@ -86,9 +86,7 @@ class JobMonitor implements Gridmix.Comp
* Add a running job's status to the polling queue.
*/
public void add(JobStats job) throws InterruptedException {
- synchronized (runningJobs) {
runningJobs.put(job);
- }
}
/**
@@ -147,12 +145,10 @@ class JobMonitor implements Gridmix.Comp
boolean shutdown;
while (true) {
try {
- synchronized (runningJobs) {
- synchronized (mJobs) {
- graceful = JobMonitor.this.graceful;
- shutdown = JobMonitor.this.shutdown;
- runningJobs.drainTo(mJobs);
- }
+ synchronized (mJobs) {
+ graceful = JobMonitor.this.graceful;
+ shutdown = JobMonitor.this.shutdown;
+ runningJobs.drainTo(mJobs);
}
// shutdown conditions; either shutdown requested and all jobs
@@ -160,11 +156,9 @@ class JobMonitor implements Gridmix.Comp
// submitted jobs not in the monitored set
if (shutdown) {
if (!graceful) {
- synchronized (runningJobs) {
- while (!runningJobs.isEmpty()) {
- synchronized (mJobs) {
- runningJobs.drainTo(mJobs);
- }
+ while (!runningJobs.isEmpty()) {
+ synchronized (mJobs) {
+ runningJobs.drainTo(mJobs);
}
}
break;
Modified: hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SerialJobFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SerialJobFactory.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SerialJobFactory.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SerialJobFactory.java Fri Apr 12 23:05:28 2013
@@ -175,4 +175,8 @@ public class SerialJobFactory extends Jo
LOG.info(" Starting Serial submission ");
this.rThread.start();
}
+ // it is need for test
+ void setDistCacheEmulator(DistributedCacheEmulator e) {
+ jobCreator.setDistCacheEmulator(e);
+ }
}
Modified: hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SleepJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SleepJob.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SleepJob.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SleepJob.java Fri Apr 12 23:05:28 2013
@@ -84,7 +84,7 @@ public class SleepJob extends GridmixJob
String[] hosts) throws IOException {
super(conf, submissionMillis, jobdesc, outRoot, ugi, seq);
this.fakeLocations = numLocations;
- this.hosts = hosts;
+ this.hosts = hosts.clone();
this.selector = (fakeLocations > 0)? new Selector(hosts.length, (float) fakeLocations
/ hosts.length, rand.get()) : null;
this.mapTasksOnly = conf.getBoolean(SLEEPJOB_MAPTASK_ONLY, false);
@@ -289,9 +289,9 @@ public class SleepJob extends GridmixJob
this.id = id;
this.sleepDuration = sleepDuration;
nSpec = reduceDurations.length;
- this.reduceDurations = reduceDurations;
+ this.reduceDurations = reduceDurations.clone();
this.nMaps = nMaps;
- this.locations = locations;
+ this.locations = locations.clone();
}
@Override
@@ -349,7 +349,7 @@ public class SleepJob extends GridmixJob
@Override
public String[] getLocations() {
- return locations;
+ return locations.clone();
}
}
Modified: hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java Fri Apr 12 23:05:28 2013
@@ -93,7 +93,7 @@ implements ResourceUsageEmulatorPlugin {
private static final float DEFAULT_HEAP_LOAD_RATIO = 0.1F;
- public static int ONE_MB = 1024 * 1024;
+ public static final int ONE_MB = 1024 * 1024;
/**
* Defines the core heap usage emulation algorithm. This engine is expected
@@ -129,7 +129,8 @@ implements ResourceUsageEmulatorPlugin {
public static class DefaultHeapUsageEmulator
implements HeapUsageEmulatorCore {
// store the unit loads in a list
- protected static ArrayList<Object> heapSpace = new ArrayList<Object>();
+ protected static final ArrayList<Object> heapSpace =
+ new ArrayList<Object>();
/**
* Increase heap usage by current process by the given amount.
Modified: hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobFactory.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobFactory.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobFactory.java Fri Apr 12 23:05:28 2013
@@ -29,13 +29,13 @@ import java.util.concurrent.CountDownLat
/**
* Component generating random job traces for testing on a single node.
*/
-class DebugJobFactory {
+public class DebugJobFactory {
interface Debuggable {
ArrayList<JobStory> getSubmitted();
}
- public static JobFactory getFactory(
+ public static JobFactory<?> getFactory(
JobSubmitter submitter, Path scratch, int numJobs, Configuration conf,
CountDownLatch startFlag, UserResolver resolver) throws IOException {
GridmixJobSubmissionPolicy policy = GridmixJobSubmissionPolicy.getPolicy(
Modified: hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobProducer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobProducer.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobProducer.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobProducer.java Fri Apr 12 23:05:28 2013
@@ -216,7 +216,7 @@ public class DebugJobProducer implements
if (user == null) {
user = String.format("foobar%d", id);
}
- GridmixTestUtils.createHomeAndStagingDirectory(user, (JobConf)conf);
+ GridmixTestUtils.createHomeAndStagingDirectory(user, conf);
return user;
}
@@ -300,7 +300,7 @@ public class DebugJobProducer implements
@Override
public String getQueueName() {
- String qName = "q" + ((id % 2) + 1);
+ String qName = "default";
return qName;
}
Modified: hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/GridmixTestUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/GridmixTestUtils.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/GridmixTestUtils.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/GridmixTestUtils.java Fri Apr 12 23:05:28 2013
@@ -4,55 +4,76 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.mapred.MiniMRCluster;
-import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MiniMRClientCluster;
+import org.apache.hadoop.mapred.MiniMRClientClusterFactory;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
-import org.apache.hadoop.security.Groups;
import java.io.IOException;
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
* Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
*/
public class GridmixTestUtils {
private static final Log LOG = LogFactory.getLog(GridmixTestUtils.class);
static final Path DEST = new Path("/gridmix");
static FileSystem dfs = null;
static MiniDFSCluster dfsCluster = null;
- static MiniMRCluster mrCluster = null;
+ static MiniMRClientCluster mrvl = null;
+ protected static final String GRIDMIX_USE_QUEUE_IN_TRACE =
+ "gridmix.job-submission.use-queue-in-trace";
+ protected static final String GRIDMIX_DEFAULT_QUEUE =
+ "gridmix.job-submission.default-queue";
- public static void initCluster() throws IOException {
+ public static void initCluster(Class<?> caller) throws IOException {
Configuration conf = new Configuration();
- conf.set("mapred.queue.names", "default,q1,q2");
- dfsCluster = new MiniDFSCluster(conf, 3, true, null);
+// conf.set("mapred.queue.names", "default,q1,q2");
+ conf.set("mapred.queue.names", "default");
+ conf.set("yarn.scheduler.capacity.root.queues", "default");
+ conf.set("yarn.scheduler.capacity.root.default.capacity", "100.0");
+
+
+ conf.setBoolean(GRIDMIX_USE_QUEUE_IN_TRACE, false);
+ conf.set(GRIDMIX_DEFAULT_QUEUE, "default");
+
+
+ dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true)
+ .build();// MiniDFSCluster(conf, 3, true, null);
dfs = dfsCluster.getFileSystem();
conf.set(JTConfig.JT_RETIREJOBS, "false");
- mrCluster = new MiniMRCluster(3, dfs.getUri().toString(), 1, null, null,
- new JobConf(conf));
+ mrvl = MiniMRClientClusterFactory.create(caller, 2, conf);
+
+ conf = mrvl.getConfig();
+ String[] files = conf.getStrings(MRJobConfig.CACHE_FILES);
+ if (files != null) {
+ String[] timestamps = new String[files.length];
+ for (int i = 0; i < files.length; i++) {
+ timestamps[i] = Long.toString(System.currentTimeMillis());
+ }
+ conf.setStrings(MRJobConfig.CACHE_FILE_TIMESTAMPS, timestamps);
+ }
+
}
public static void shutdownCluster() throws IOException {
- if (mrCluster != null) {
- mrCluster.shutdown();
+ if (mrvl != null) {
+ mrvl.stop();
}
if (dfsCluster != null) {
dfsCluster.shutdown();
@@ -61,23 +82,25 @@ public class GridmixTestUtils {
/**
* Methods to generate the home directory for dummy users.
- *
+ *
* @param conf
*/
- public static void createHomeAndStagingDirectory(String user, JobConf conf) {
+ public static void createHomeAndStagingDirectory(String user,
+ Configuration conf) {
try {
FileSystem fs = dfsCluster.getFileSystem();
String path = "/user/" + user;
Path homeDirectory = new Path(path);
- if(fs.exists(homeDirectory)) {
- fs.delete(homeDirectory,true);
- }
- LOG.info("Creating Home directory : " + homeDirectory);
- fs.mkdirs(homeDirectory);
- changePermission(user,homeDirectory, fs);
- Path stagingArea =
- new Path(conf.get("mapreduce.jobtracker.staging.root.dir",
- "/tmp/hadoop/mapred/staging"));
+ if (!fs.exists(homeDirectory)) {
+ LOG.info("Creating Home directory : " + homeDirectory);
+ fs.mkdirs(homeDirectory);
+ changePermission(user, homeDirectory, fs);
+
+ }
+ changePermission(user, homeDirectory, fs);
+ Path stagingArea = new Path(
+ conf.get("mapreduce.jobtracker.staging.root.dir",
+ "/tmp/hadoop/mapred/staging"));
LOG.info("Creating Staging root directory : " + stagingArea);
fs.mkdirs(stagingArea);
fs.setPermission(stagingArea, new FsPermission((short) 0777));
@@ -87,7 +110,7 @@ public class GridmixTestUtils {
}
static void changePermission(String user, Path homeDirectory, FileSystem fs)
- throws IOException {
+ throws IOException {
fs.setOwner(homeDirectory, user, "");
}
}
Modified: hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java Fri Apr 12 23:05:28 2013
@@ -133,7 +133,7 @@ public class TestGridmixSummary {
/**
* A fake {@link JobFactory}.
*/
- @SuppressWarnings("unchecked")
+ @SuppressWarnings("rawtypes")
private static class FakeJobFactory extends JobFactory {
/**
* A fake {@link JobStoryProducer} for {@link FakeJobFactory}.
@@ -167,7 +167,7 @@ public class TestGridmixSummary {
* Test {@link ExecutionSummarizer}.
*/
@Test
- @SuppressWarnings("unchecked")
+ @SuppressWarnings({ "unchecked", "rawtypes" })
public void testExecutionSummarizer() throws IOException {
Configuration conf = new Configuration();
@@ -360,7 +360,6 @@ public class TestGridmixSummary {
* Test {@link ClusterSummarizer}.
*/
@Test
- @SuppressWarnings("deprecation")
public void testClusterSummarizer() throws IOException {
ClusterSummarizer cs = new ClusterSummarizer();
Configuration conf = new Configuration();
@@ -374,13 +373,13 @@ public class TestGridmixSummary {
assertEquals("JT name mismatch", jt, cs.getJobTrackerInfo());
assertEquals("NN name mismatch", nn, cs.getNamenodeInfo());
- ClusterStats cstats = ClusterStats.getClusterStats();
+ ClusterStats cStats = ClusterStats.getClusterStats();
conf.set(JTConfig.JT_IPC_ADDRESS, "local");
conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "local");
JobClient jc = new JobClient(conf);
- cstats.setClusterMetric(jc.getClusterStatus());
+ cStats.setClusterMetric(jc.getClusterStatus());
- cs.update(cstats);
+ cs.update(cStats);
// test
assertEquals("Cluster summary test failed!", 1, cs.getMaxMapTasks());
Modified: hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/Job20LineHistoryEventEmitter.java Fri Apr 12 23:05:28 2013
@@ -92,6 +92,10 @@ public class Job20LineHistoryEventEmitte
if (workflowAdjacencies == null) {
workflowAdjacencies = "";
}
+ String workflowTags = line.get("WORKFLOW_TAGS");
+ if (workflowTags == null) {
+ workflowTags = "";
+ }
if (submitTime != null) {
@@ -104,7 +108,8 @@ public class Job20LineHistoryEventEmitte
new HashMap<JobACL, AccessControlList>();
return new JobSubmittedEvent(jobID, jobName, user,
that.originalSubmitTime, jobConf, jobACLs, jobQueueName,
- workflowId, workflowName, workflowNodeName, workflowAdjacencies);
+ workflowId, workflowName, workflowNodeName, workflowAdjacencies,
+ workflowTags);
}
return null;
Modified: hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/pom.xml?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/pom.xml (original)
+++ hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/pom.xml Fri Apr 12 23:05:28 2013
@@ -127,6 +127,20 @@
</target>
</configuration>
</execution>
+ <execution>
+ <id>copy-test-bin</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <copy todir="target/bin">
+ <fileset dir="src/test/bin" />
+ </copy>
+ </target>
+ </configuration>
+ </execution>
</executions>
</plugin>
<plugin>
Modified: hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java Fri Apr 12 23:05:28 2013
@@ -294,8 +294,7 @@ public class StreamJob implements Tool {
for (String file : values) {
packageFiles_.add(file);
try {
- URI pathURI = new URI(file);
- Path path = new Path(pathURI);
+ Path path = new Path(file);
FileSystem localFs = FileSystem.getLocal(config_);
String finalPath = path.makeQualified(localFs).toString();
if(fileList.length() > 0) {
@@ -875,7 +874,7 @@ public class StreamJob implements Tool {
IdentifierResolver.TEXT_ID));
jobConf_.setClass("stream.map.output.reader.class",
idResolver.getOutputReaderClass(), OutputReader.class);
- if (isMapperACommand) {
+ if (isMapperACommand || jobConf_.get("stream.map.output") != null) {
// if mapper is a command, then map output key/value classes come from the
// idResolver
jobConf_.setMapOutputKeyClass(idResolver.getOutputKeyClass());
@@ -891,7 +890,7 @@ public class StreamJob implements Tool {
IdentifierResolver.TEXT_ID));
jobConf_.setClass("stream.reduce.output.reader.class",
idResolver.getOutputReaderClass(), OutputReader.class);
- if (isReducerACommand) {
+ if (isReducerACommand || jobConf_.get("stream.reduce.output") != null) {
// if reducer is a command, then output key/value classes come from the
// idResolver
jobConf_.setOutputKeyClass(idResolver.getOutputKeyClass());
Modified: hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java Fri Apr 12 23:05:28 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.streaming;
import java.io.DataOutputStream;
+import java.io.File;
import java.io.IOException;
import java.util.Map;
@@ -27,6 +28,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.MiniMRCluster;
+import org.apache.hadoop.util.Shell;
import org.junit.After;
import org.junit.Before;
@@ -45,7 +47,8 @@ public class TestFileArgs extends TestSt
private static final String EXPECTED_OUTPUT =
"job.jar\t\nsidefile\t\n";
- private static final String LS_PATH = "/bin/ls";
+ private static final String LS_PATH = Shell.WINDOWS ? "cmd /c dir /B" :
+ "/bin/ls";
public TestFileArgs() throws IOException
{
@@ -58,6 +61,7 @@ public class TestFileArgs extends TestSt
map = LS_PATH;
FileSystem.setDefaultUri(conf, "hdfs://" + namenode);
+ setTestDir(new File("/tmp/TestFileArgs"));
}
@Before
Modified: hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java Fri Apr 12 23:05:28 2013
@@ -70,8 +70,8 @@ public class TestMultipleArchiveFiles ex
namenode = fileSys.getUri().getAuthority();
mr = new MiniMRCluster(1, namenode, 1);
- map = "xargs cat";
- reduce = "cat";
+ map = XARGS_CAT;
+ reduce = CAT;
}
@Override
@@ -84,7 +84,8 @@ public class TestMultipleArchiveFiles ex
{
fileSys.delete(new Path(INPUT_DIR), true);
DataOutputStream dos = fileSys.create(new Path(INPUT_FILE));
- String inputFileString = "symlink1/cacheArchive1\nsymlink2/cacheArchive2";
+ String inputFileString = "symlink1" + File.separator
+ + "cacheArchive1\nsymlink2" + File.separator + "cacheArchive2";
dos.write(inputFileString.getBytes("UTF-8"));
dos.close();
Modified: hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java Fri Apr 12 23:05:28 2013
@@ -49,8 +49,8 @@ public class TestMultipleCachefiles
String CACHE_FILE = "/testing-streaming/cache.txt";
String CACHE_FILE_2 = "/testing-streaming/cache2.txt";
String input = "check to see if we can read this none reduce";
- String map = "xargs cat ";
- String reduce = "cat";
+ String map = TestStreaming.XARGS_CAT;
+ String reduce = TestStreaming.CAT;
String mapString = "testlink";
String mapString2 = "testlink2";
String cacheString = "This is just the cache string";
Modified: hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java Fri Apr 12 23:05:28 2013
@@ -33,7 +33,7 @@ public class TestStreamXmlRecordReader e
INPUT_FILE = new File("target/input.xml");
input = "<xmltag>\t\nroses.are.red\t\nviolets.are.blue\t\n" +
"bunnies.are.pink\t\n</xmltag>\t\n";
- map = "cat";
+ map = CAT;
reduce = "NONE";
outputExpect = input;
}
Modified: hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreaming.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreaming.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreaming.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreaming.java Fri Apr 12 23:05:28 2013
@@ -33,7 +33,7 @@ import static org.junit.Assert.*;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.Configuration;
-
+import org.apache.hadoop.util.Shell;
/**
* This class tests hadoopStreaming in MapReduce local mode.
@@ -43,6 +43,22 @@ public class TestStreaming
public static final String STREAMING_JAR = JarFinder.getJar(StreamJob.class);
+ /**
+ * cat command used for copying stdin to stdout as mapper or reducer function.
+ * On Windows, use a cmd script that approximates the functionality of cat.
+ */
+ static final String CAT = Shell.WINDOWS ?
+ "cmd /c " + new File("target/bin/cat.cmd").getAbsolutePath() : "cat";
+
+ /**
+ * Command used for iterating through file names on stdin and copying each
+ * file's contents to stdout, used as mapper or reducer function. On Windows,
+ * use a cmd script that approximates the functionality of xargs cat.
+ */
+ static final String XARGS_CAT = Shell.WINDOWS ?
+ "cmd /c " + new File("target/bin/xargs_cat.cmd").getAbsolutePath() :
+ "xargs cat";
+
// "map" command: grep -E (red|green|blue)
// reduce command: uniq
protected File TEST_DIR;
@@ -66,9 +82,22 @@ public class TestStreaming
UtilTest utilTest = new UtilTest(getClass().getName());
utilTest.checkUserDir();
utilTest.redirectIfAntJunit();
- TEST_DIR = new File("target/TestStreaming").getAbsoluteFile();
- OUTPUT_DIR = new File(TEST_DIR, "out");
- INPUT_FILE = new File(TEST_DIR, "input.txt");
+ setTestDir(new File("target/TestStreaming").getAbsoluteFile());
+ }
+
+ /**
+ * Sets root of test working directory and resets any other paths that must be
+ * children of the test working directory. Typical usage is for subclasses
+ * that use HDFS to override the test directory to the form "/tmp/<test name>"
+ * so that on Windows, tests won't attempt to use paths containing a ':' from
+ * the drive specifier. The ':' character is considered invalid by HDFS.
+ *
+ * @param testDir File to set
+ */
+ protected void setTestDir(File testDir) {
+ TEST_DIR = testDir;
+ OUTPUT_DIR = new File(testDir, "out");
+ INPUT_FILE = new File(testDir, "input.txt");
}
@Before
@@ -89,19 +118,18 @@ public class TestStreaming
protected void createInput() throws IOException
{
- DataOutputStream out = getFileSystem().create(
- new Path(INPUT_FILE.getAbsolutePath()));
+ DataOutputStream out = getFileSystem().create(new Path(
+ INPUT_FILE.getPath()));
out.write(getInputData().getBytes("UTF-8"));
out.close();
}
protected void setInputOutput() {
- inputFile = INPUT_FILE.getAbsolutePath();
- outDir = OUTPUT_DIR.getAbsolutePath();
+ inputFile = INPUT_FILE.getPath();
+ outDir = OUTPUT_DIR.getPath();
}
protected String[] genArgs() {
- setInputOutput();
args.add("-input");args.add(inputFile);
args.add("-output");args.add(outDir);
args.add("-mapper");args.add(map);
@@ -129,7 +157,7 @@ public class TestStreaming
}
protected void checkOutput() throws IOException {
- Path outPath = new Path(OUTPUT_DIR.getAbsolutePath(), "part-00000");
+ Path outPath = new Path(OUTPUT_DIR.getPath(), "part-00000");
FileSystem fs = getFileSystem();
String output = StreamUtil.slurpHadoop(outPath, fs);
fs.delete(outPath, true);
@@ -155,6 +183,7 @@ public class TestStreaming
* @throws IOException
*/
protected int runStreamJob() throws IOException {
+ setInputOutput();
createInput();
boolean mayExit = false;
Modified: hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingKeyValue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingKeyValue.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingKeyValue.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingKeyValue.java Fri Apr 12 23:05:28 2013
@@ -76,7 +76,7 @@ public class TestStreamingKeyValue
return new String[] {
"-input", INPUT_FILE.getAbsolutePath(),
"-output", OUTPUT_DIR.getAbsolutePath(),
- "-mapper", "cat",
+ "-mapper", TestStreaming.CAT,
"-jobconf", MRJobConfig.PRESERVE_FAILED_TASK_FILES + "=true",
"-jobconf", "stream.non.zero.exit.is.failure=true",
"-jobconf", "stream.tmpdir="+System.getProperty("test.build.data","/tmp"),
Modified: hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java Fri Apr 12 23:05:28 2013
@@ -24,6 +24,7 @@ import org.apache.hadoop.mapred.MapReduc
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.junit.Before;
import org.junit.Test;
@@ -119,7 +120,7 @@ public class TestStreamingOutputKeyValue
@Test
public void testJavaMapperAndCommandReducer() throws Exception {
map = "org.apache.hadoop.mapred.lib.IdentityMapper";
- reduce = "cat";
+ reduce = CAT;
super.testCommandLine();
}
@@ -127,7 +128,7 @@ public class TestStreamingOutputKeyValue
@Test
public void testJavaMapperAndCommandReducerAndZeroReduces() throws Exception {
map = "org.apache.hadoop.mapred.lib.IdentityMapper";
- reduce = "cat";
+ reduce = CAT;
args.add("-numReduceTasks");
args.add("0");
super.testCommandLine();
@@ -136,7 +137,7 @@ public class TestStreamingOutputKeyValue
// Check with Command Mapper, Java Reducer
@Test
public void testCommandMapperAndJavaReducer() throws Exception {
- map = "cat";
+ map = CAT;
reduce = MyReducer.class.getName();
super.testCommandLine();
}
@@ -144,7 +145,7 @@ public class TestStreamingOutputKeyValue
// Check with Command Mapper, Java Reducer and -numReduceTasks 0
@Test
public void testCommandMapperAndJavaReducerAndZeroReduces() throws Exception {
- map = "cat";
+ map = CAT;
reduce = MyReducer.class.getName();
args.add("-numReduceTasks");
args.add("0");
@@ -154,7 +155,7 @@ public class TestStreamingOutputKeyValue
// Check with Command Mapper, Reducer = "NONE"
@Test
public void testCommandMapperWithReduceNone() throws Exception {
- map = "cat";
+ map = CAT;
reduce = "NONE";
super.testCommandLine();
}
@@ -162,8 +163,8 @@ public class TestStreamingOutputKeyValue
// Check with Command Mapper, Command Reducer
@Test
public void testCommandMapperAndCommandReducer() throws Exception {
- map = "cat";
- reduce = "cat";
+ map = CAT;
+ reduce = CAT;
super.testCommandLine();
}
@@ -171,12 +172,23 @@ public class TestStreamingOutputKeyValue
@Test
public void testCommandMapperAndCommandReducerAndZeroReduces()
throws Exception {
- map = "cat";
- reduce = "cat";
+ map = CAT;
+ reduce = CAT;
args.add("-numReduceTasks");
args.add("0");
super.testCommandLine();
}
+
+ @Test
+ public void testDefaultToIdentityReducer() throws Exception {
+ args.add("-mapper");args.add(map);
+ args.add("-jobconf");
+ args.add("mapreduce.task.files.preserve.failedtasks=true");
+ args.add("-jobconf");
+ args.add("stream.tmpdir="+System.getProperty("test.build.data","/tmp"));
+ args.add("-inputformat");args.add(TextInputFormat.class.getName());
+ super.testCommandLine();
+ }
@Override
@Test
Modified: hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java Fri Apr 12 23:05:28 2013
@@ -83,7 +83,7 @@ public class TestStreamingTaskLog {
* (b) hadoop.tasklog.totalLogFileSize
* for the children of java tasks in streaming jobs.
*/
- @Test (timeout = 30000)
+ @Test (timeout = 120000)
public void testStreamingTaskLogWithHadoopCmd() {
try {
final int numSlaves = 1;
@@ -95,13 +95,14 @@ public class TestStreamingTaskLog {
fs.delete(testDir, true);
}
fs.mkdirs(testDir);
- File scriptFile = createScript(
- testDir.toString() + "/testTaskLog.sh");
+ File scriptFile = createScript(testDir.toString() +
+ (Shell.WINDOWS ? "/testTaskLog.cmd" : "/testTaskLog.sh"));
conf.setBoolean(JTConfig.JT_PERSIST_JOBSTATUS, false);
mr = new MiniMRCluster(numSlaves, fs.getUri().toString(), 1, null, null, conf);
writeInputFile(fs, inputPath);
- map = scriptFile.getAbsolutePath();
+ map = Shell.WINDOWS ? "cmd /c " + scriptFile.getAbsolutePath() :
+ scriptFile.getAbsolutePath();
runStreamJobAndValidateEnv();
@@ -120,8 +121,12 @@ public class TestStreamingTaskLog {
File scriptFile = new File(script);
UtilTest.recursiveDelete(scriptFile);
FileOutputStream in = new FileOutputStream(scriptFile);
- in.write(("cat > /dev/null 2>&1\n" +
- "echo $HADOOP_ROOT_LOGGER $HADOOP_CLIENT_OPTS").getBytes());
+ if (Shell.WINDOWS) {
+ in.write("@echo %HADOOP_ROOT_LOGGER% %HADOOP_CLIENT_OPTS%".getBytes());
+ } else {
+ in.write(("cat > /dev/null 2>&1\n" +
+ "echo $HADOOP_ROOT_LOGGER $HADOOP_CLIENT_OPTS").getBytes());
+ }
in.close();
Shell.execCommand(Shell.getSetPermissionCommand("+x", false,
Modified: hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java Fri Apr 12 23:05:28 2013
@@ -47,13 +47,13 @@ public class TestSymLink
String OUTPUT_DIR = "/testing-streaming/out";
String CACHE_FILE = "/testing-streaming/cache.txt";
String input = "check to see if we can read this none reduce";
- String map = "xargs cat ";
- String reduce = "cat";
+ String map = TestStreaming.XARGS_CAT;
+ String reduce = TestStreaming.CAT;
String mapString = "testlink\n";
String cacheString = "This is just the cache string";
StreamJob job;
- @Test (timeout = 60000)
+ @Test (timeout = 120000)
public void testSymLink() throws Exception
{
boolean mayExit = false;
Modified: hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java Fri Apr 12 23:05:28 2013
@@ -43,7 +43,6 @@ public class TrApp
// Note the dots translated to underscore:
// property names have been escaped in PipeMapRed.safeEnvVarName()
expectDefined("mapreduce_cluster_local_dir");
- expect("mapred_output_format_class", "org.apache.hadoop.mapred.TextOutputFormat");
expect("mapreduce_map_output_key_class", "org.apache.hadoop.io.Text");
expect("mapreduce_map_output_value_class", "org.apache.hadoop.io.Text");