You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/08 05:36:14 UTC
svn commit: r1079107 - in /hadoop/common/branches/yahoo-merge: CHANGES.txt
src/java/org/apache/hadoop/fs/FileUtil.java
src/test/core/org/apache/hadoop/fs/TestFileUtil.java
Author: omalley
Date: Tue Mar 8 04:36:14 2011
New Revision: 1079107
URL: http://svn.apache.org/viewvc?rev=1079107&view=rev
Log:
commit a54282a74ea5989719b8f84d5f8f87c2b1d374ca
Author: Douglass Cutting <cu...@apache.org>
Date: Tue Oct 26 21:15:36 2010 +0000
HADOOP-7006. Fix 'fs -getmerge' command to not be a no-op. Contributed by Chris Nauroth.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1027748 13f79535-47bb-0310-9956-ffa450edef68
Modified:
hadoop/common/branches/yahoo-merge/CHANGES.txt
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FileUtil.java
hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/fs/TestFileUtil.java
Modified: hadoop/common/branches/yahoo-merge/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/CHANGES.txt?rev=1079107&r1=1079106&r2=1079107&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/CHANGES.txt (original)
+++ hadoop/common/branches/yahoo-merge/CHANGES.txt Tue Mar 8 04:36:14 2011
@@ -277,6 +277,9 @@ Trunk (unreleased changes)
HADOOP-6947. Kerberos relogin should set refreshKrb5Config to true.
(Todd Lipcon via tomwhite)
+ HADOOP-7006. Fix 'fs -getmerge' command to not be a no-op.
+ (Chris Nauroth via cutting)
+
Release 0.21.1 - Unreleased
IMPROVEMENTS
Modified: hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FileUtil.java?rev=1079107&r1=1079106&r2=1079107&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FileUtil.java (original)
+++ hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FileUtil.java Tue Mar 8 04:36:14 2011
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs;
import java.io.*;
+import java.util.Arrays;
import java.util.Enumeration;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
@@ -279,13 +280,14 @@ public class FileUtil {
Configuration conf, String addString) throws IOException {
dstFile = checkDest(srcDir.getName(), dstFS, dstFile, false);
- if (srcFS.getFileStatus(srcDir).isDirectory())
+ if (!srcFS.getFileStatus(srcDir).isDirectory())
return false;
OutputStream out = dstFS.create(dstFile);
try {
FileStatus contents[] = srcFS.listStatus(srcDir);
+ Arrays.sort(contents);
for (int i = 0; i < contents.length; i++) {
if (contents[i].isFile()) {
InputStream in = srcFS.open(contents[i].getPath());
Modified: hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/fs/TestFileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/fs/TestFileUtil.java?rev=1079107&r1=1079106&r2=1079107&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/fs/TestFileUtil.java (original)
+++ hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/fs/TestFileUtil.java Tue Mar 8 04:36:14 2011
@@ -17,14 +17,18 @@
*/
package org.apache.hadoop.fs;
+import java.io.BufferedReader;
import java.io.File;
+import java.io.FileReader;
import java.io.IOException;
+import java.io.PrintWriter;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
@@ -32,8 +36,9 @@ import org.junit.Test;
public class TestFileUtil {
private static final Log LOG = LogFactory.getLog(TestFileUtil.class);
- final static private File TEST_DIR = new File(System.getProperty(
- "test.build.data", "/tmp"), "fu");
+ private static final String TEST_ROOT_DIR = System.getProperty(
+ "test.build.data", "/tmp") + "/fu";
+ private static final File TEST_DIR = new File(TEST_ROOT_DIR);
private static String FILE = "x";
private static String LINK = "y";
private static String DIR = "dir";
@@ -41,9 +46,10 @@ public class TestFileUtil {
private File tmp = new File(TEST_DIR, "tmp");
private File dir1 = new File(del, DIR + "1");
private File dir2 = new File(del, DIR + "2");
+ private File partitioned = new File(TEST_DIR, "partitioned");
/**
- * Creates directories del and tmp for testing.
+ * Creates multiple directories for testing.
*
* Contents of them are
* dir:tmp:
@@ -54,12 +60,17 @@ public class TestFileUtil {
* dir: dir2 : file:x
* link: y to tmp/x
* link: tmpDir to tmp
+ * dir:partitioned:
+ * file: part-r-00000, contents: "foo"
+ * file: part-r-00001, contents: "bar"
*/
private void setupDirs() throws IOException {
Assert.assertFalse(del.exists());
Assert.assertFalse(tmp.exists());
+ Assert.assertFalse(partitioned.exists());
del.mkdirs();
tmp.mkdirs();
+ partitioned.mkdirs();
new File(del, FILE).createNewFile();
File tmpFile = new File(tmp, FILE);
tmpFile.createNewFile();
@@ -78,12 +89,39 @@ public class TestFileUtil {
File linkDir = new File(del, "tmpDir");
FileUtil.symLink(tmp.toString(), linkDir.toString());
Assert.assertEquals(5, del.listFiles().length);
+
+ // create files in partitioned directories
+ createFile(partitioned, "part-r-00000", "foo");
+ createFile(partitioned, "part-r-00001", "bar");
+ }
+
+ /**
+ * Creates a new file in the specified directory, with the specified name and
+ * the specified file contents. This method will add a newline terminator to
+ * the end of the contents string in the destination file.
+ * @param directory File non-null destination directory.
+ * @param name String non-null file name.
+ * @param contents String non-null file contents.
+ * @throws IOException if an I/O error occurs.
+ */
+ private void createFile(File directory, String name, String contents)
+ throws IOException {
+ File newFile = new File(directory, name);
+ PrintWriter pw = new PrintWriter(newFile);
+
+ try {
+ pw.println(contents);
+ }
+ finally {
+ pw.close();
+ }
}
@After
public void tearDown() throws IOException {
FileUtil.fullyDelete(del);
FileUtil.fullyDelete(tmp);
+ FileUtil.fullyDelete(partitioned);
}
@Test
@@ -312,4 +350,58 @@ public class TestFileUtil {
boolean ret = FileUtil.fullyDeleteContents(new MyFile(del));
validateAndSetWritablePermissions(ret);
}
+
+ @Test
+ public void testCopyMergeSingleDirectory() throws IOException {
+ setupDirs();
+ boolean copyMergeResult = copyMerge("partitioned", "tmp/merged");
+ Assert.assertTrue("Expected successful copyMerge result.", copyMergeResult);
+ File merged = new File(TEST_DIR, "tmp/merged");
+ Assert.assertTrue("File tmp/merged must exist after copyMerge.",
+ merged.exists());
+ BufferedReader rdr = new BufferedReader(new FileReader(merged));
+
+ try {
+ Assert.assertEquals("Line 1 of merged file must contain \"foo\".",
+ "foo", rdr.readLine());
+ Assert.assertEquals("Line 2 of merged file must contain \"bar\".",
+ "bar", rdr.readLine());
+ Assert.assertNull("Expected end of file reading merged file.",
+ rdr.readLine());
+ }
+ finally {
+ rdr.close();
+ }
+ }
+
+ /**
+ * Calls FileUtil.copyMerge using the specified source and destination paths.
+ * Both source and destination are assumed to be on the local file system.
+ * The call will not delete source on completion and will not add an
+ * additional string between files.
+ * @param src String non-null source path.
+ * @param dst String non-null destination path.
+ * @return boolean true if the call to FileUtil.copyMerge was successful.
+ * @throws IOException if an I/O error occurs.
+ */
+ private boolean copyMerge(String src, String dst)
+ throws IOException {
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.getLocal(conf);
+ final boolean result;
+
+ try {
+ Path srcPath = new Path(TEST_ROOT_DIR, src);
+ Path dstPath = new Path(TEST_ROOT_DIR, dst);
+ boolean deleteSource = false;
+ String addString = null;
+ result = FileUtil.copyMerge(fs, srcPath, fs, dstPath, deleteSource, conf,
+ addString);
+ }
+ finally {
+ fs.close();
+ }
+
+ return result;
+ }
}