You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cm...@apache.org on 2014/04/18 18:32:43 UTC
svn commit: r1588509 - in /hadoop/common/branches/HADOOP-10388: ./
hadoop-project/ hadoop-project/src/site/
hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/
hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/
hadoo...
Author: cmccabe
Date: Fri Apr 18 16:32:35 2014
New Revision: 1588509
URL: http://svn.apache.org/r1588509
Log:
Merge trunk into the HADOOP-10388 branch
Added:
hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
- copied unchanged from r1588387, hadoop/common/trunk/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/
- copied from r1588387, hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/
Removed:
hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java
Modified:
hadoop/common/branches/HADOOP-10388/ (props changed)
hadoop/common/branches/HADOOP-10388/BUILDING.txt
hadoop/common/branches/HADOOP-10388/hadoop-project/pom.xml
hadoop/common/branches/HADOOP-10388/hadoop-project/src/site/site.xml
hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java
hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestFileBasedCopyListing.java
hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestGlobbedCopyListing.java
hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java
hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java
hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java
hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StateDeserializer.java
hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java
hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-streaming/dev-support/findbugs-exclude.xml
Propchange: hadoop/common/branches/HADOOP-10388/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk:r1582150-1588387
Modified: hadoop/common/branches/HADOOP-10388/BUILDING.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/BUILDING.txt?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/BUILDING.txt (original)
+++ hadoop/common/branches/HADOOP-10388/BUILDING.txt Fri Apr 18 16:32:35 2014
@@ -9,6 +9,8 @@ Requirements:
* Findbugs 1.3.9 (if running findbugs)
* ProtocolBuffer 2.5.0
* CMake 2.6 or newer (if compiling native code)
+* Zlib devel (if compiling native code)
+* openssl devel ( if compiling native hadoop-pipes )
* Internet connection for first build (to fetch all Maven and Hadoop dependencies)
----------------------------------------------------------------------------------
@@ -189,6 +191,7 @@ Requirements:
* ProtocolBuffer 2.5.0
* Windows SDK or Visual Studio 2010 Professional
* Unix command-line tools from GnuWin32 or Cygwin: sh, mkdir, rm, cp, tar, gzip
+* zlib headers (if building native code bindings for zlib)
* Internet connection for first build (to fetch all Maven and Hadoop dependencies)
If using Visual Studio, it must be Visual Studio 2010 Professional (not 2012).
@@ -228,6 +231,18 @@ native code is built by enabling the 'na
is enabled by default when building on Windows since the native components
are required (not optional) on Windows.
+If native code bindings for zlib are required, then the zlib headers must be
+deployed on the build machine. Set the ZLIB_HOME environment variable to the
+directory containing the headers.
+
+set ZLIB_HOME=C:\zlib-1.2.7
+
+At runtime, zlib1.dll must be accessible on the PATH. Hadoop has been tested
+with zlib 1.2.7, built using Visual Studio 2010 out of contrib\vstudio\vc10 in
+the zlib 1.2.7 source tree.
+
+http://www.zlib.net/
+
----------------------------------------------------------------------------------
Building distributions:
Modified: hadoop/common/branches/HADOOP-10388/hadoop-project/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-project/pom.xml?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-project/pom.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-project/pom.xml Fri Apr 18 16:32:35 2014
@@ -631,22 +631,22 @@
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
- <version>1.8.8</version>
+ <version>1.9.13</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
- <version>1.8.8</version>
+ <version>1.9.13</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-jaxrs</artifactId>
- <version>1.8.8</version>
+ <version>1.9.13</version>
</dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-xc</artifactId>
- <version>1.8.8</version>
+ <version>1.9.13</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
Modified: hadoop/common/branches/HADOOP-10388/hadoop-project/src/site/site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-project/src/site/site.xml?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-project/src/site/site.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-project/src/site/site.xml Fri Apr 18 16:32:35 2014
@@ -98,6 +98,7 @@
<item name="YARN Architecture" href="hadoop-yarn/hadoop-yarn-site/YARN.html"/>
<item name="Capacity Scheduler" href="hadoop-yarn/hadoop-yarn-site/CapacityScheduler.html"/>
<item name="Fair Scheduler" href="hadoop-yarn/hadoop-yarn-site/FairScheduler.html"/>
+ <item name="ResourceManager Restart" href="hadoop-yarn/hadoop-yarn-site/ResourceManagerRestart.html"/>
<item name="Web Application Proxy" href="hadoop-yarn/hadoop-yarn-site/WebApplicationProxy.html"/>
<item name="YARN Timeline Server" href="hadoop-yarn/hadoop-yarn-site/TimelineServer.html"/>
<item name="Writing YARN Applications" href="hadoop-yarn/hadoop-yarn-site/WritingYarnApplications.html"/>
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java Fri Apr 18 16:32:35 2014
@@ -40,6 +40,8 @@ import org.apache.hadoop.util.ToolRunner
import java.io.IOException;
import java.util.Random;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* DistCp is the main driver-class for DistCpV2.
* For command-line use, DistCp::main() orchestrates the parsing of command-line
@@ -87,7 +89,8 @@ public class DistCp extends Configured i
/**
* To be used with the ToolRunner. Not for public consumption.
*/
- private DistCp() {}
+ @VisibleForTesting
+ public DistCp() {}
/**
* Implementation of Tool::run(). Orchestrates the copy of source file(s)
@@ -105,7 +108,7 @@ public class DistCp extends Configured i
try {
inputOptions = (OptionsParser.parse(argv));
-
+ setTargetPathExists();
LOG.info("Input Options: " + inputOptions);
} catch (Throwable e) {
LOG.error("Invalid arguments: ", e);
@@ -170,6 +173,18 @@ public class DistCp extends Configured i
}
/**
+ * Set targetPathExists in both inputOptions and job config,
+ * for the benefit of CopyCommitter
+ */
+ private void setTargetPathExists() throws IOException {
+ Path target = inputOptions.getTargetPath();
+ FileSystem targetFS = target.getFileSystem(getConf());
+ boolean targetExists = targetFS.exists(target);
+ inputOptions.setTargetPathExists(targetExists);
+ getConf().setBoolean(DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS,
+ targetExists);
+ }
+ /**
* Create Job object for submitting it, with all the configuration
*
* @return Reference to job object.
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java Fri Apr 18 16:32:35 2014
@@ -74,6 +74,9 @@ public class DistCpConstants {
*/
public static final String CONF_LABEL_TARGET_FINAL_PATH = "distcp.target.final.path";
+ /* Boolean to indicate whether the target of distcp exists. */
+ public static final String CONF_LABEL_TARGET_PATH_EXISTS = "distcp.target.path.exists";
+
/**
* DistCp job id for consumers of the Disctp
*/
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java Fri Apr 18 16:32:35 2014
@@ -60,6 +60,10 @@ public class DistCpOptions {
private Path targetPath;
+ // targetPathExist is a derived field, it's initialized in the
+ // beginning of distcp.
+ private boolean targetPathExists = true;
+
public static enum FileAttribute{
REPLICATION, BLOCKSIZE, USER, GROUP, PERMISSION, CHECKSUMTYPE;
@@ -123,6 +127,7 @@ public class DistCpOptions {
this.sourceFileListing = that.getSourceFileListing();
this.sourcePaths = that.getSourcePaths();
this.targetPath = that.getTargetPath();
+ this.targetPathExists = that.getTargetPathExists();
}
}
@@ -439,6 +444,22 @@ public class DistCpOptions {
return targetPath;
}
+ /**
+ * Getter for the targetPathExists.
+ * @return The target-path.
+ */
+ public boolean getTargetPathExists() {
+ return targetPathExists;
+ }
+
+ /**
+ * Set targetPathExists.
+ * @param targetPathExists Whether the target path of distcp exists.
+ */
+ public boolean setTargetPathExists(boolean targetPathExists) {
+ return this.targetPathExists = targetPathExists;
+ }
+
public void validate(DistCpOptionSwitch option, boolean value) {
boolean syncFolder = (option == DistCpOptionSwitch.SYNC_FOLDERS ?
@@ -515,6 +536,7 @@ public class DistCpOptions {
", sourceFileListing=" + sourceFileListing +
", sourcePaths=" + sourcePaths +
", targetPath=" + targetPath +
+ ", targetPathExists=" + targetPathExists +
'}';
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java Fri Apr 18 16:32:35 2014
@@ -111,7 +111,24 @@ public class SimpleCopyListing extends C
public void doBuildListing(Path pathToListingFile, DistCpOptions options) throws IOException {
doBuildListing(getWriter(pathToListingFile), options);
}
-
+ /**
+ * Collect the list of
+ * <sourceRelativePath, sourceFileStatus>
+ * to be copied and write to the sequence file. In essence, any file or
+ * directory that need to be copied or sync-ed is written as an entry to the
+ * sequence file, with the possible exception of the source root:
+ * when either -update (sync) or -overwrite switch is specified, and if
+ * the the source root is a directory, then the source root entry is not
+ * written to the sequence file, because only the contents of the source
+ * directory need to be copied in this case.
+ * See {@link org.apache.hadoop.tools.util.DistCpUtils.getRelativePath} for
+ * how relative path is computed.
+ * See computeSourceRootPath method for how the root path of the source is
+ * computed.
+ * @param fileListWriter
+ * @param options
+ * @throws IOException
+ */
@VisibleForTesting
public void doBuildListing(SequenceFile.Writer fileListWriter,
DistCpOptions options) throws IOException {
@@ -125,7 +142,12 @@ public class SimpleCopyListing extends C
boolean localFile = (rootStatus.getClass() != FileStatus.class);
FileStatus[] sourceFiles = sourceFS.listStatus(path);
- if (sourceFiles != null && sourceFiles.length > 0) {
+ boolean explore = (sourceFiles != null && sourceFiles.length > 0);
+ if (!explore || rootStatus.isDirectory()) {
+ writeToFileListingRoot(fileListWriter, rootStatus, sourcePathRoot,
+ localFile, options);
+ }
+ if (explore) {
for (FileStatus sourceStatus: sourceFiles) {
if (LOG.isDebugEnabled()) {
LOG.debug("Recording source-path: " + sourceStatus.getPath() + " for copy.");
@@ -141,9 +163,6 @@ public class SimpleCopyListing extends C
localFile, options);
}
}
- } else {
- writeToFileListing(fileListWriter, rootStatus, sourcePathRoot,
- localFile, options);
}
}
fileListWriter.close();
@@ -158,18 +177,19 @@ public class SimpleCopyListing extends C
Path target = options.getTargetPath();
FileSystem targetFS = target.getFileSystem(getConf());
+ final boolean targetPathExists = options.getTargetPathExists();
boolean solitaryFile = options.getSourcePaths().size() == 1
&& !sourceStatus.isDirectory();
if (solitaryFile) {
- if (targetFS.isFile(target) || !targetFS.exists(target)) {
+ if (targetFS.isFile(target) || !targetPathExists) {
return sourceStatus.getPath();
} else {
return sourceStatus.getPath().getParent();
}
} else {
- boolean specialHandling = (options.getSourcePaths().size() == 1 && !targetFS.exists(target)) ||
+ boolean specialHandling = (options.getSourcePaths().size() == 1 && !targetPathExists) ||
options.shouldSyncFolder() || options.shouldOverwrite();
return specialHandling && sourceStatus.isDirectory() ? sourceStatus.getPath() :
@@ -253,15 +273,30 @@ public class SimpleCopyListing extends C
}
}
}
+
+ private void writeToFileListingRoot(SequenceFile.Writer fileListWriter,
+ FileStatus fileStatus, Path sourcePathRoot,
+ boolean localFile,
+ DistCpOptions options) throws IOException {
+ boolean syncOrOverwrite = options.shouldSyncFolder() ||
+ options.shouldOverwrite();
+ if (fileStatus.getPath().equals(sourcePathRoot) &&
+ fileStatus.isDirectory() && syncOrOverwrite) {
+ // Skip the root-paths when syncOrOverwrite
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skip " + fileStatus.getPath());
+ }
+ return;
+ }
+ writeToFileListing(fileListWriter, fileStatus, sourcePathRoot, localFile,
+ options);
+ }
private void writeToFileListing(SequenceFile.Writer fileListWriter,
FileStatus fileStatus,
Path sourcePathRoot,
boolean localFile,
DistCpOptions options) throws IOException {
- if (fileStatus.getPath().equals(sourcePathRoot) && fileStatus.isDirectory())
- return; // Skip the root-paths.
-
if (LOG.isDebugEnabled()) {
LOG.debug("REL PATH: " + DistCpUtils.getRelativePath(sourcePathRoot,
fileStatus.getPath()) + ", FULL PATH: " + fileStatus.getPath());
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java Fri Apr 18 16:32:35 2014
@@ -54,7 +54,10 @@ public class CopyCommitter extends FileO
private static final Log LOG = LogFactory.getLog(CopyCommitter.class);
private final TaskAttemptContext taskAttemptContext;
-
+ private boolean syncFolder = false;
+ private boolean overwrite = false;
+ private boolean targetPathExists = true;
+
/**
* Create a output committer
*
@@ -71,6 +74,10 @@ public class CopyCommitter extends FileO
@Override
public void commitJob(JobContext jobContext) throws IOException {
Configuration conf = jobContext.getConfiguration();
+ syncFolder = conf.getBoolean(DistCpConstants.CONF_LABEL_SYNC_FOLDERS, false);
+ overwrite = conf.getBoolean(DistCpConstants.CONF_LABEL_OVERWRITE, false);
+ targetPathExists = conf.getBoolean(DistCpConstants.CONF_LABEL_TARGET_PATH_EXISTS, true);
+
super.commitJob(jobContext);
cleanupTempFiles(jobContext);
@@ -155,6 +162,8 @@ public class CopyCommitter extends FileO
// user/group permissions, etc.) based on the corresponding source directories.
private void preserveFileAttributesForDirectories(Configuration conf) throws IOException {
String attrSymbols = conf.get(DistCpConstants.CONF_LABEL_PRESERVE_STATUS);
+ final boolean syncOrOverwrite = syncFolder || overwrite;
+
LOG.info("About to preserve attributes: " + attrSymbols);
EnumSet<FileAttribute> attributes = DistCpUtils.unpackAttributes(attrSymbols);
@@ -179,12 +188,10 @@ public class CopyCommitter extends FileO
if (! srcFileStatus.isDirectory()) continue;
Path targetFile = new Path(targetRoot.toString() + "/" + srcRelPath);
-
- // Skip the root folder.
- // Status can't be preserved on root-folder. (E.g. multiple paths may
- // be copied to a single target folder. Which source-attributes to use
- // on the target is undefined.)
- if (targetRoot.equals(targetFile)) continue;
+ //
+ // Skip the root folder when syncOrOverwrite is true.
+ //
+ if (targetRoot.equals(targetFile) && syncOrOverwrite) continue;
FileSystem targetFS = targetFile.getFileSystem(conf);
DistCpUtils.preserve(targetFS, targetFile, srcFileStatus, attributes);
@@ -218,7 +225,14 @@ public class CopyCommitter extends FileO
Path targetFinalPath = new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH));
targets.add(targetFinalPath);
DistCpOptions options = new DistCpOptions(targets, new Path("/NONE"));
-
+ //
+ // Set up options to be the same from the CopyListing.buildListing's perspective,
+ // so to collect similar listings as when doing the copy
+ //
+ options.setOverwrite(overwrite);
+ options.setSyncFolder(syncFolder);
+ options.setTargetPathExists(targetPathExists);
+
target.buildListing(targetListing, options);
Path sortedTargetListing = DistCpUtils.sortListing(clusterFS, conf, targetListing);
long totalLen = clusterFS.getFileStatus(sortedTargetListing).getLen();
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java Fri Apr 18 16:32:35 2014
@@ -103,12 +103,14 @@ public class TestCopyListing extends Sim
DistCpOptions options = new DistCpOptions(srcPaths, target);
Path listingFile = new Path("/tmp/list4");
listing.buildListing(listingFile, options);
- Assert.assertEquals(listing.getNumberOfPaths(), 2);
+ Assert.assertEquals(listing.getNumberOfPaths(), 3);
SequenceFile.Reader reader = new SequenceFile.Reader(getConf(),
SequenceFile.Reader.file(listingFile));
FileStatus fileStatus = new FileStatus();
Text relativePath = new Text();
Assert.assertTrue(reader.next(relativePath, fileStatus));
+ Assert.assertEquals(relativePath.toString(), "/1");
+ Assert.assertTrue(reader.next(relativePath, fileStatus));
Assert.assertEquals(relativePath.toString(), "/1/file");
Assert.assertTrue(reader.next(relativePath, fileStatus));
Assert.assertEquals(relativePath.toString(), "/2");
@@ -270,7 +272,8 @@ public class TestCopyListing extends Sim
final Path listFile = new Path(testRoot, "/tmp/fileList.seq");
listing.buildListing(listFile, options);
- reader = new SequenceFile.Reader(fs, listFile, getConf());
+ reader = new SequenceFile.Reader(getConf(), SequenceFile.Reader.file(listFile));
+
FileStatus fileStatus = new FileStatus();
Text relativePath = new Text();
Assert.assertTrue(reader.next(relativePath, fileStatus));
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java Fri Apr 18 16:32:35 2014
@@ -85,7 +85,7 @@ public class TestDistCpViewFs {
addEntries(listFile, "singlefile1/file1");
createFiles("singlefile1/file1");
- runTest(listFile, target, sync);
+ runTest(listFile, target, false, sync);
checkResult(target, 1);
} catch (IOException e) {
@@ -108,7 +108,7 @@ public class TestDistCpViewFs {
addEntries(listFile, "singlefile1/file1");
createFiles("singlefile1/file1", target.toString());
- runTest(listFile, target, sync);
+ runTest(listFile, target, false, sync);
checkResult(target, 1);
} catch (IOException e) {
@@ -132,7 +132,7 @@ public class TestDistCpViewFs {
createFiles("singlefile2/file2");
mkdirs(target.toString());
- runTest(listFile, target, sync);
+ runTest(listFile, target, true, sync);
checkResult(target, 1, "file2");
} catch (IOException e) {
@@ -155,7 +155,7 @@ public class TestDistCpViewFs {
addEntries(listFile, "singledir");
mkdirs(root + "/singledir/dir1");
- runTest(listFile, target, sync);
+ runTest(listFile, target, false, sync);
checkResult(target, 1, "dir1");
} catch (IOException e) {
@@ -174,7 +174,7 @@ public class TestDistCpViewFs {
mkdirs(root + "/singledir/dir1");
mkdirs(target.toString());
- runTest(listFile, target, false);
+ runTest(listFile, target, true, false);
checkResult(target, 1, "singledir/dir1");
} catch (IOException e) {
@@ -193,7 +193,7 @@ public class TestDistCpViewFs {
mkdirs(root + "/Usingledir/Udir1");
mkdirs(target.toString());
- runTest(listFile, target, true);
+ runTest(listFile, target, true, true);
checkResult(target, 1, "Udir1");
} catch (IOException e) {
@@ -217,7 +217,7 @@ public class TestDistCpViewFs {
createFiles("multifile/file3", "multifile/file4", "multifile/file5");
mkdirs(target.toString());
- runTest(listFile, target, sync);
+ runTest(listFile, target, true, sync);
checkResult(target, 3, "file3", "file4", "file5");
} catch (IOException e) {
@@ -240,7 +240,7 @@ public class TestDistCpViewFs {
addEntries(listFile, "multifile/file3", "multifile/file4", "multifile/file5");
createFiles("multifile/file3", "multifile/file4", "multifile/file5");
- runTest(listFile, target, sync);
+ runTest(listFile, target, false, sync);
checkResult(target, 3, "file3", "file4", "file5");
} catch (IOException e) {
@@ -259,7 +259,7 @@ public class TestDistCpViewFs {
createFiles("multifile/file3", "multifile/file4", "multifile/file5");
mkdirs(target.toString(), root + "/singledir/dir1");
- runTest(listFile, target, false);
+ runTest(listFile, target, true, false);
checkResult(target, 2, "multifile/file3", "multifile/file4", "multifile/file5", "singledir/dir1");
} catch (IOException e) {
@@ -278,7 +278,7 @@ public class TestDistCpViewFs {
createFiles("Umultifile/Ufile3", "Umultifile/Ufile4", "Umultifile/Ufile5");
mkdirs(target.toString(), root + "/Usingledir/Udir1");
- runTest(listFile, target, true);
+ runTest(listFile, target, true, true);
checkResult(target, 4, "Ufile3", "Ufile4", "Ufile5", "Udir1");
} catch (IOException e) {
@@ -297,7 +297,7 @@ public class TestDistCpViewFs {
createFiles("multifile/file3", "multifile/file4", "multifile/file5");
mkdirs(root + "/singledir/dir1");
- runTest(listFile, target, false);
+ runTest(listFile, target, false, false);
checkResult(target, 2, "multifile/file3", "multifile/file4",
"multifile/file5", "singledir/dir1");
@@ -317,7 +317,7 @@ public class TestDistCpViewFs {
createFiles("multifile/file3", "multifile/file4", "multifile/file5");
mkdirs(root + "/singledir/dir1");
- runTest(listFile, target, true);
+ runTest(listFile, target, false, true);
checkResult(target, 4, "file3", "file4", "file5", "dir1");
} catch (IOException e) {
@@ -338,7 +338,7 @@ public class TestDistCpViewFs {
createFiles("multifile/file3", "multifile/file4", "multifile/file5");
createFiles("singledir/dir2/file6");
- runTest(listFile, target, false);
+ runTest(listFile, target, false, false);
checkResult(target, 2, "multifile/file3", "multifile/file4", "multifile/file5",
"singledir/dir2/file6");
@@ -361,7 +361,7 @@ public class TestDistCpViewFs {
createFiles("multifile/file3", "multifile/file4", "multifile/file5");
createFiles("singledir/dir2/file6");
- runTest(listFile, target, true);
+ runTest(listFile, target, false, true);
checkResult(target, 4, "file3", "file4", "file5", "dir2/file6");
} catch (IOException e) {
@@ -384,7 +384,7 @@ public class TestDistCpViewFs {
createFiles("singledir1/dir3/file7", "singledir1/dir3/file8",
"singledir1/dir3/file9");
- runTest(listFile, target, false);
+ runTest(listFile, target, false, false);
checkResult(target, 4, "file3", "file4", "file5",
"dir3/file7", "dir3/file8", "dir3/file9");
@@ -408,7 +408,7 @@ public class TestDistCpViewFs {
createFiles("singledir1/dir3/file7", "singledir1/dir3/file8",
"singledir1/dir3/file9");
- runTest(listFile, target, true);
+ runTest(listFile, target, false, true);
checkResult(target, 6, "file3", "file4", "file5",
"file7", "file8", "file9");
@@ -460,9 +460,11 @@ public class TestDistCpViewFs {
}
}
- private void runTest(Path listFile, Path target, boolean sync) throws IOException {
+ private void runTest(Path listFile, Path target, boolean targetExists,
+ boolean sync) throws IOException {
DistCpOptions options = new DistCpOptions(listFile, target);
options.setSyncFolder(sync);
+ options.setTargetPathExists(targetExists);
try {
new DistCp(getConf(), options).execute();
} catch (Exception e) {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestFileBasedCopyListing.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestFileBasedCopyListing.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestFileBasedCopyListing.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestFileBasedCopyListing.java Fri Apr 18 16:32:35 2014
@@ -112,7 +112,7 @@ public class TestFileBasedCopyListing {
addEntries(listFile, "/tmp/singlefile1/file1");
createFiles("/tmp/singlefile1/file1");
- runTest(listFile, target, sync);
+ runTest(listFile, target, false, sync);
checkResult(listFile, 0);
} catch (IOException e) {
@@ -138,7 +138,7 @@ public class TestFileBasedCopyListing {
addEntries(listFile, "/tmp/singlefile1/file1");
createFiles("/tmp/singlefile1/file1", target.toString());
- runTest(listFile, target, sync);
+ runTest(listFile, target, false, sync);
checkResult(listFile, 0);
} catch (IOException e) {
@@ -165,7 +165,7 @@ public class TestFileBasedCopyListing {
createFiles("/tmp/singlefile2/file2");
mkdirs(target.toString());
- runTest(listFile, target, sync);
+ runTest(listFile, target, true, sync);
checkResult(listFile, 1);
} catch (IOException e) {
@@ -191,7 +191,7 @@ public class TestFileBasedCopyListing {
addEntries(listFile, "/tmp/singledir");
mkdirs("/tmp/singledir/dir1");
- runTest(listFile, target, sync);
+ runTest(listFile, target, false, sync);
checkResult(listFile, 1);
} catch (IOException e) {
@@ -213,7 +213,7 @@ public class TestFileBasedCopyListing {
mkdirs("/tmp/singledir/dir1");
mkdirs(target.toString());
- runTest(listFile, target);
+ runTest(listFile, target, true);
checkResult(listFile, 1);
} catch (IOException e) {
@@ -235,7 +235,7 @@ public class TestFileBasedCopyListing {
mkdirs("/tmp/Usingledir/Udir1");
mkdirs(target.toString());
- runTest(listFile, target, true);
+ runTest(listFile, target, true, true);
checkResult(listFile, 1);
} catch (IOException e) {
@@ -262,7 +262,7 @@ public class TestFileBasedCopyListing {
createFiles("/tmp/multifile/file3", "/tmp/multifile/file4", "/tmp/multifile/file5");
mkdirs(target.toString());
- runTest(listFile, target, sync);
+ runTest(listFile, target, true, sync);
checkResult(listFile, 3);
} catch (IOException e) {
@@ -288,7 +288,7 @@ public class TestFileBasedCopyListing {
addEntries(listFile, "/tmp/multifile/file3", "/tmp/multifile/file4", "/tmp/multifile/file5");
createFiles("/tmp/multifile/file3", "/tmp/multifile/file4", "/tmp/multifile/file5");
- runTest(listFile, target, sync);
+ runTest(listFile, target, false, sync);
checkResult(listFile, 3);
} catch (IOException e) {
@@ -310,7 +310,7 @@ public class TestFileBasedCopyListing {
createFiles("/tmp/multifile/file3", "/tmp/multifile/file4", "/tmp/multifile/file5");
mkdirs(target.toString(), "/tmp/singledir/dir1");
- runTest(listFile, target);
+ runTest(listFile, target, true);
checkResult(listFile, 4);
} catch (IOException e) {
@@ -440,7 +440,7 @@ public class TestFileBasedCopyListing {
"/tmp/singledir1/dir3/file9");
mkdirs(target.toString());
- runTest(listFile, target);
+ runTest(listFile, target, true);
checkResult(listFile, 6);
} catch (IOException e) {
@@ -507,14 +507,17 @@ public class TestFileBasedCopyListing {
}
}
- private void runTest(Path listFile, Path target) throws IOException {
- runTest(listFile, target, true);
+ private void runTest(Path listFile, Path target,
+ boolean targetExists) throws IOException {
+ runTest(listFile, target, targetExists, true);
}
- private void runTest(Path listFile, Path target, boolean sync) throws IOException {
+ private void runTest(Path listFile, Path target, boolean targetExists,
+ boolean sync) throws IOException {
CopyListing listing = new FileBasedCopyListing(config, CREDENTIALS);
DistCpOptions options = new DistCpOptions(listFile, target);
options.setSyncFolder(sync);
+ options.setTargetPathExists(targetExists);
listing.buildListing(listFile, options);
}
@@ -530,6 +533,11 @@ public class TestFileBasedCopyListing {
Text relPath = new Text();
FileStatus fileStatus = new FileStatus();
while (reader.next(relPath, fileStatus)) {
+ if (fileStatus.isDirectory() && relPath.toString().equals("")) {
+ // ignore root with empty relPath, which is an entry to be
+ // used for preserving root attributes etc.
+ continue;
+ }
Assert.assertEquals(fileStatus.getPath().toUri().getPath(), map.get(relPath.toString()));
recCount++;
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestGlobbedCopyListing.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestGlobbedCopyListing.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestGlobbedCopyListing.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestGlobbedCopyListing.java Fri Apr 18 16:32:35 2014
@@ -111,7 +111,7 @@ public class TestGlobbedCopyListing {
Path target = new Path(fileSystemPath.toString() + "/tmp/target");
Path listingPath = new Path(fileSystemPath.toString() + "/tmp/META/fileList.seq");
DistCpOptions options = new DistCpOptions(Arrays.asList(source), target);
-
+ options.setTargetPathExists(false);
new GlobbedCopyListing(new Configuration(), CREDENTIALS).buildListing(listingPath, options);
verifyContents(listingPath);
@@ -124,6 +124,11 @@ public class TestGlobbedCopyListing {
FileStatus value = new FileStatus();
Map<String, String> actualValues = new HashMap<String, String>();
while (reader.next(key, value)) {
+ if (value.isDirectory() && key.toString().equals("")) {
+ // ignore root with empty relPath, which is an entry to be
+ // used for preserving root attributes etc.
+ continue;
+ }
actualValues.put(value.getPath().toString(), key.toString());
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java Fri Apr 18 16:32:35 2014
@@ -82,7 +82,7 @@ public class TestIntegration {
addEntries(listFile, "singlefile1/file1");
createFiles("singlefile1/file1");
- runTest(listFile, target, sync);
+ runTest(listFile, target, false, sync);
checkResult(target, 1);
} catch (IOException e) {
@@ -105,7 +105,7 @@ public class TestIntegration {
addEntries(listFile, "singlefile1/file1");
createFiles("singlefile1/file1", "target");
- runTest(listFile, target, sync);
+ runTest(listFile, target, false, sync);
checkResult(target, 1);
} catch (IOException e) {
@@ -129,7 +129,7 @@ public class TestIntegration {
createFiles("singlefile2/file2");
mkdirs(target.toString());
- runTest(listFile, target, sync);
+ runTest(listFile, target, true, sync);
checkResult(target, 1, "file2");
} catch (IOException e) {
@@ -152,7 +152,7 @@ public class TestIntegration {
addEntries(listFile, "singledir");
mkdirs(root + "/singledir/dir1");
- runTest(listFile, target, sync);
+ runTest(listFile, target, false, sync);
checkResult(target, 1, "dir1");
} catch (IOException e) {
@@ -171,7 +171,7 @@ public class TestIntegration {
mkdirs(root + "/singledir/dir1");
mkdirs(target.toString());
- runTest(listFile, target, false);
+ runTest(listFile, target, true, false);
checkResult(target, 1, "singledir/dir1");
} catch (IOException e) {
@@ -190,7 +190,7 @@ public class TestIntegration {
mkdirs(root + "/Usingledir/Udir1");
mkdirs(target.toString());
- runTest(listFile, target, true);
+ runTest(listFile, target, true, true);
checkResult(target, 1, "Udir1");
} catch (IOException e) {
@@ -214,7 +214,7 @@ public class TestIntegration {
createFiles("multifile/file3", "multifile/file4", "multifile/file5");
mkdirs(target.toString());
- runTest(listFile, target, sync);
+ runTest(listFile, target, true, sync);
checkResult(target, 3, "file3", "file4", "file5");
} catch (IOException e) {
@@ -286,7 +286,7 @@ public class TestIntegration {
addEntries(listFile, "multifile/file3", "multifile/file4", "multifile/file5");
createFiles("multifile/file3", "multifile/file4", "multifile/file5");
- runTest(listFile, target, sync);
+ runTest(listFile, target, false, sync);
checkResult(target, 3, "file3", "file4", "file5");
} catch (IOException e) {
@@ -305,7 +305,7 @@ public class TestIntegration {
createFiles("multifile/file3", "multifile/file4", "multifile/file5");
mkdirs(target.toString(), root + "/singledir/dir1");
- runTest(listFile, target, false);
+ runTest(listFile, target, true, false);
checkResult(target, 2, "multifile/file3", "multifile/file4", "multifile/file5", "singledir/dir1");
} catch (IOException e) {
@@ -324,7 +324,7 @@ public class TestIntegration {
createFiles("Umultifile/Ufile3", "Umultifile/Ufile4", "Umultifile/Ufile5");
mkdirs(target.toString(), root + "/Usingledir/Udir1");
- runTest(listFile, target, true);
+ runTest(listFile, target, true, true);
checkResult(target, 4, "Ufile3", "Ufile4", "Ufile5", "Udir1");
} catch (IOException e) {
@@ -343,7 +343,7 @@ public class TestIntegration {
createFiles("multifile/file3", "multifile/file4", "multifile/file5");
mkdirs(root + "/singledir/dir1");
- runTest(listFile, target, false);
+ runTest(listFile, target, false, false);
checkResult(target, 2, "multifile/file3", "multifile/file4",
"multifile/file5", "singledir/dir1");
@@ -363,7 +363,7 @@ public class TestIntegration {
createFiles("multifile/file3", "multifile/file4", "multifile/file5");
mkdirs(root + "/singledir/dir1");
- runTest(listFile, target, true);
+ runTest(listFile, target, false, true);
checkResult(target, 4, "file3", "file4", "file5", "dir1");
} catch (IOException e) {
@@ -382,7 +382,7 @@ public class TestIntegration {
createFiles("srcdir/file1", "dstdir/file1", "dstdir/file2");
Path target = new Path(root + "/dstdir");
- runTest(listFile, target, true, true, false);
+ runTest(listFile, target, false, true, true, false);
checkResult(target, 1, "file1");
} catch (IOException e) {
@@ -406,7 +406,7 @@ public class TestIntegration {
createWithContents("dstdir/file1", contents2);
Path target = new Path(root + "/dstdir");
- runTest(listFile, target, false, false, true);
+ runTest(listFile, target, false, false, false, true);
checkResult(target, 1, "file1");
@@ -436,7 +436,7 @@ public class TestIntegration {
createFiles("multifile/file3", "multifile/file4", "multifile/file5");
createFiles("singledir/dir2/file6");
- runTest(listFile, target, false);
+ runTest(listFile, target, false, false);
checkResult(target, 2, "multifile/file3", "multifile/file4", "multifile/file5",
"singledir/dir2/file6");
@@ -459,7 +459,7 @@ public class TestIntegration {
createFiles("multifile/file3", "multifile/file4", "multifile/file5");
createFiles("singledir/dir2/file6");
- runTest(listFile, target, true);
+ runTest(listFile, target, false, true);
checkResult(target, 4, "file3", "file4", "file5", "dir2/file6");
} catch (IOException e) {
@@ -482,7 +482,7 @@ public class TestIntegration {
createFiles("singledir1/dir3/file7", "singledir1/dir3/file8",
"singledir1/dir3/file9");
- runTest(listFile, target, false);
+ runTest(listFile, target, false, false);
checkResult(target, 4, "file3", "file4", "file5",
"dir3/file7", "dir3/file8", "dir3/file9");
@@ -506,7 +506,7 @@ public class TestIntegration {
createFiles("singledir1/dir3/file7", "singledir1/dir3/file8",
"singledir1/dir3/file9");
- runTest(listFile, target, true);
+ runTest(listFile, target, false, true);
checkResult(target, 6, "file3", "file4", "file5",
"file7", "file8", "file9");
@@ -584,16 +584,19 @@ public class TestIntegration {
}
}
- private void runTest(Path listFile, Path target, boolean sync) throws IOException {
- runTest(listFile, target, sync, false, false);
+ private void runTest(Path listFile, Path target, boolean targetExists,
+ boolean sync) throws IOException {
+ runTest(listFile, target, targetExists, sync, false, false);
}
- private void runTest(Path listFile, Path target, boolean sync, boolean delete,
+ private void runTest(Path listFile, Path target, boolean targetExists,
+ boolean sync, boolean delete,
boolean overwrite) throws IOException {
DistCpOptions options = new DistCpOptions(listFile, target);
options.setSyncFolder(sync);
options.setDeleteMissing(delete);
options.setOverwrite(overwrite);
+ options.setTargetPathExists(targetExists);
try {
new DistCp(getConf(), options).execute();
} catch (Exception e) {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java Fri Apr 18 16:32:35 2014
@@ -354,7 +354,7 @@ public class TestOptionsParser {
DistCpOptions option = new DistCpOptions(new Path("abc"), new Path("xyz"));
String val = "DistCpOptions{atomicCommit=false, syncFolder=false, deleteMissing=false, " +
"ignoreFailures=false, maxMaps=20, sslConfigurationFile='null', copyStrategy='uniformsize', " +
- "sourceFileListing=abc, sourcePaths=null, targetPath=xyz}";
+ "sourceFileListing=abc, sourcePaths=null, targetPath=xyz, targetPathExists=true}";
Assert.assertEquals(val, option.toString());
Assert.assertNotSame(DistCpOptionSwitch.ATOMIC_COMMIT.toString(),
DistCpOptionSwitch.ATOMIC_COMMIT.name());
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java Fri Apr 18 16:32:35 2014
@@ -146,7 +146,8 @@ public class TestCopyCommitter {
new Path("/out"));
options.preserve(FileAttribute.PERMISSION);
options.appendToConf(conf);
-
+ options.setTargetPathExists(false);
+
CopyListing listing = new GlobbedCopyListing(conf, CREDENTIALS);
Path listingFile = new Path("/tmp1/" + String.valueOf(rand.nextLong()));
listing.buildListing(listingFile, options);
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java Fri Apr 18 16:32:35 2014
@@ -313,7 +313,7 @@ public class TestCopyMapper {
= stubContext.getContext();
Configuration configuration = context.getConfiguration();
- String workPath = new Path("hftp://localhost:1234/*/*/*/?/")
+ String workPath = new Path("webhdfs://localhost:1234/*/*/*/?/")
.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString();
configuration.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH,
workPath);
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java Fri Apr 18 16:32:35 2014
@@ -133,7 +133,9 @@ public class TestUniformSizeInputFormat
Path sourcePath = recordReader.getCurrentValue().getPath();
FileSystem fs = sourcePath.getFileSystem(configuration);
FileStatus fileStatus [] = fs.listStatus(sourcePath);
- Assert.assertEquals(fileStatus.length, 1);
+ if (fileStatus.length > 1) {
+ continue;
+ }
currentSplitSize += fileStatus[0].getLen();
}
Assert.assertTrue(
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java Fri Apr 18 16:32:35 2014
@@ -59,7 +59,9 @@ public class TestDynamicInputFormat {
for (int i=0; i<N_FILES; ++i)
createFile("/tmp/source/" + String.valueOf(i));
-
+ FileSystem fileSystem = cluster.getFileSystem();
+ expectedFilePaths.add(fileSystem.listStatus(
+ new Path("/tmp/source/0"))[0].getPath().getParent().toString());
}
private static Configuration getConfigurationForCluster() {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-extras/src/test/java/org/apache/hadoop/tools/TestCopyFiles.java Fri Apr 18 16:32:35 2014
@@ -891,64 +891,6 @@ public class TestCopyFiles extends TestC
return home;
}
- public void testHftpAccessControl() throws Exception {
- MiniDFSCluster cluster = null;
- try {
- final UserGroupInformation DFS_UGI = createUGI("dfs", true);
- final UserGroupInformation USER_UGI = createUGI("user", false);
-
- //start cluster by DFS_UGI
- final Configuration dfsConf = new Configuration();
- cluster = new MiniDFSCluster.Builder(dfsConf).numDataNodes(2).build();
- cluster.waitActive();
-
- final String httpAdd = dfsConf.get("dfs.http.address");
- final URI nnURI = FileSystem.getDefaultUri(dfsConf);
- final String nnUri = nnURI.toString();
- FileSystem fs1 = DFS_UGI.doAs(new PrivilegedExceptionAction<FileSystem>() {
- public FileSystem run() throws IOException {
- return FileSystem.get(nnURI, dfsConf);
- }
- });
- final Path home =
- createHomeDirectory(fs1, USER_UGI);
-
- //now, login as USER_UGI
- final Configuration userConf = new Configuration();
- final FileSystem fs =
- USER_UGI.doAs(new PrivilegedExceptionAction<FileSystem>() {
- public FileSystem run() throws IOException {
- return FileSystem.get(nnURI, userConf);
- }
- });
-
- final Path srcrootpath = new Path(home, "src_root");
- final String srcrootdir = srcrootpath.toString();
- final Path dstrootpath = new Path(home, "dst_root");
- final String dstrootdir = dstrootpath.toString();
- final DistCpV1 distcp = USER_UGI.doAs(new PrivilegedExceptionAction<DistCpV1>() {
- public DistCpV1 run() {
- return new DistCpV1(userConf);
- }
- });
-
- FileSystem.mkdirs(fs, srcrootpath, new FsPermission((short)0700));
- final String[] args = {"hftp://"+httpAdd+srcrootdir, nnUri+dstrootdir};
-
- { //copy with permission 000, should fail
- fs.setPermission(srcrootpath, new FsPermission((short)0));
- USER_UGI.doAs(new PrivilegedExceptionAction<Void>() {
- public Void run() throws Exception {
- assertEquals(-3, ToolRunner.run(distcp, args));
- return null;
- }
- });
- }
- } finally {
- if (cluster != null) { cluster.shutdown(); }
- }
- }
-
/** test -delete */
public void testDelete() throws Exception {
final Configuration conf = new Configuration();
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StateDeserializer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StateDeserializer.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StateDeserializer.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StateDeserializer.java Fri Apr 18 16:32:35 2014
@@ -24,7 +24,7 @@ import org.codehaus.jackson.JsonParser;
import org.codehaus.jackson.JsonProcessingException;
import org.codehaus.jackson.map.DeserializationContext;
import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.deser.StdDeserializer;
+import org.codehaus.jackson.map.deser.std.StdDeserializer;
import org.codehaus.jackson.node.ObjectNode;
/**
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java Fri Apr 18 16:32:35 2014
@@ -120,7 +120,7 @@ public class RumenToSLSConverter {
Writer output = new FileWriter(outputFile);
try {
ObjectMapper mapper = new ObjectMapper();
- ObjectWriter writer = mapper.defaultPrettyPrintingWriter();
+ ObjectWriter writer = mapper.writerWithDefaultPrettyPrinter();
Iterator<Map> i = mapper.readValues(
new JsonFactory().createJsonParser(input), Map.class);
while (i.hasNext()) {
@@ -141,7 +141,7 @@ public class RumenToSLSConverter {
Writer output = new FileWriter(outputFile);
try {
ObjectMapper mapper = new ObjectMapper();
- ObjectWriter writer = mapper.defaultPrettyPrintingWriter();
+ ObjectWriter writer = mapper.writerWithDefaultPrettyPrinter();
for (Map.Entry<String, Set<String>> entry : rackNodeMap.entrySet()) {
Map rack = new LinkedHashMap();
rack.put("rack", entry.getKey());
Modified: hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-streaming/dev-support/findbugs-exclude.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-streaming/dev-support/findbugs-exclude.xml?rev=1588509&r1=1588508&r2=1588509&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-streaming/dev-support/findbugs-exclude.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-tools/hadoop-streaming/dev-support/findbugs-exclude.xml Fri Apr 18 16:32:35 2014
@@ -16,6 +16,18 @@
limitations under the License.
-->
<FindBugsFilter>
+ <!-- Workaround bugs in findbugs 1.3.9. See http://sourceforge.net/p/findbugs/bugs/918 for more details. -->
+ <Match>
+ <Or>
+ <Package name="org.apache.hadoop.streaming" />
+ </Or>
+ <Bug pattern="NP_ALWAYS_NULL"/>
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.streaming.JarBuilder" />
+ <Bug pattern="NP_NULL_PARAM_DEREF_ALL_TARGETS_DANGEROUS"/>
+ </Match>
+
<Match>
<Or>
<Class name="org.apache.hadoop.streaming.PipeMapper" />
@@ -27,4 +39,13 @@
</Or>
<Bug pattern="EI_EXPOSE_REP"/>
</Match>
+
+ <Match>
+ <Package name="org.apache.hadoop.record" />
+ <Or>
+ <Bug pattern="EI_EXPOSE_REP" />
+ <Bug pattern="EI_EXPOSE_REP2" />
+ <Bug pattern="MS_PKGPROTECT" />
+ </Or>
+ </Match>
</FindBugsFilter>