You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by to...@apache.org on 2012/09/10 20:45:50 UTC
svn commit: r1383030 - in /hadoop/common/branches/HDFS-3077: ./
hadoop-client/
hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/
hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/
hadoop-tools/hadoop-distcp/src/tes...
Author: todd
Date: Mon Sep 10 18:45:45 2012
New Revision: 1383030
URL: http://svn.apache.org/viewvc?rev=1383030&view=rev
Log:
Merge trunk into QJM branch
Modified:
hadoop/common/branches/HDFS-3077/ (props changed)
hadoop/common/branches/HDFS-3077/hadoop-client/pom.xml
hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
Propchange: hadoop/common/branches/HDFS-3077/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk:r1380987-1383029
Modified: hadoop/common/branches/HDFS-3077/hadoop-client/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-client/pom.xml?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-client/pom.xml (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-client/pom.xml Mon Sep 10 18:45:45 2012
@@ -40,10 +40,6 @@
<scope>compile</scope>
<exclusions>
<exclusion>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- </exclusion>
- <exclusion>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
</exclusion>
Modified: hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java Mon Sep 10 18:45:45 2012
@@ -98,6 +98,11 @@ public class DistCp extends Configured i
* @return On success, it returns 0. Else, -1.
*/
public int run(String[] argv) {
+ if (argv.length < 1) {
+ OptionsParser.usage();
+ return DistCpConstants.INVALID_ARGUMENT;
+ }
+
try {
inputOptions = (OptionsParser.parse(argv));
Modified: hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/OptionsParser.java Mon Sep 10 18:45:45 2012
@@ -156,6 +156,10 @@ public class OptionsParser {
try {
Integer mapBandwidth = Integer.parseInt(
getVal(command, DistCpOptionSwitch.BANDWIDTH.getSwitch()).trim());
+ if (mapBandwidth.intValue() <= 0) {
+ throw new IllegalArgumentException("Bandwidth specified is not positive: " +
+ mapBandwidth);
+ }
option.setMapBandwidth(mapBandwidth);
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Bandwidth specified is invalid: " +
Modified: hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java Mon Sep 10 18:45:45 2012
@@ -255,7 +255,7 @@ public class CopyMapper extends Mapper<T
long bytesCopied;
try {
- bytesCopied = (Long)new RetriableFileCopyCommand(description)
+ bytesCopied = (Long)new RetriableFileCopyCommand(skipCrc, description)
.execute(sourceFileStatus, target, context, fileAttributes);
} catch (Exception e) {
context.setStatus("Copy Failure: " + sourceFileStatus.getPath());
Modified: hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java Mon Sep 10 18:45:45 2012
@@ -41,7 +41,8 @@ public class RetriableFileCopyCommand ex
private static Log LOG = LogFactory.getLog(RetriableFileCopyCommand.class);
private static int BUFFER_SIZE = 8 * 1024;
-
+ private boolean skipCrc = false;
+
/**
* Constructor, taking a description of the action.
* @param description Verbose description of the copy operation.
@@ -49,6 +50,17 @@ public class RetriableFileCopyCommand ex
public RetriableFileCopyCommand(String description) {
super(description);
}
+
+ /**
+ * Create a RetriableFileCopyCommand.
+ *
+ * @param skipCrc Whether to skip the crc check.
+ * @param description A verbose description of the copy operation.
+ */
+ public RetriableFileCopyCommand(boolean skipCrc, String description) {
+ this(description);
+ this.skipCrc = skipCrc;
+ }
/**
* Implementation of RetriableCommand::doExecute().
@@ -92,7 +104,7 @@ public class RetriableFileCopyCommand ex
compareFileLengths(sourceFileStatus, tmpTargetPath, configuration, bytesRead);
//At this point, src&dest lengths are same. if length==0, we skip checksum
- if (bytesRead != 0) {
+ if ((bytesRead != 0) && (!skipCrc)) {
compareCheckSums(sourceFS, sourceFileStatus.getPath(), targetFS, tmpTargetPath);
}
promoteTmpToTarget(tmpTargetPath, target, targetFS);
Modified: hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java?rev=1383030&r1=1383029&r2=1383030&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java Mon Sep 10 18:45:45 2012
@@ -110,6 +110,24 @@ public class TestOptionsParser {
"hdfs://localhost:8020/target/"});
Assert.assertEquals(options.getMapBandwidth(), 11);
}
+
+ @Test(expected=IllegalArgumentException.class)
+ public void testParseNonPositiveBandwidth() {
+ OptionsParser.parse(new String[] {
+ "-bandwidth",
+ "-11",
+ "hdfs://localhost:8020/source/first",
+ "hdfs://localhost:8020/target/"});
+ }
+
+ @Test(expected=IllegalArgumentException.class)
+ public void testParseZeroBandwidth() {
+ OptionsParser.parse(new String[] {
+ "-bandwidth",
+ "0",
+ "hdfs://localhost:8020/source/first",
+ "hdfs://localhost:8020/target/"});
+ }
@Test
public void testParseSkipCRC() {