svn commit: r1152791 - in /hadoop/common/trunk/common: CHANGES.txt src/java/org/apache/hadoop/fs/FileSystem.java src/test/core/org/apache/hadoop/fs/FSMainOperationsBaseTest.java

[sz...@apache.org]

Author: szetszwo
Date: Mon Aug  1 14:20:24 2011
New Revision: 1152791

URL: http://svn.apache.org/viewvc?rev=1152791&view=rev
Log:
HADOOP-7178. Add a parameter, useRawLocalFileSystem, to copyToLocalFile(..) in FileSystem.  Contributed by Uma Maheswara Rao G

Modified:
    hadoop/common/trunk/common/CHANGES.txt
    hadoop/common/trunk/common/src/java/org/apache/hadoop/fs/FileSystem.java
    hadoop/common/trunk/common/src/test/core/org/apache/hadoop/fs/FSMainOperationsBaseTest.java

Modified: hadoop/common/trunk/common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/common/CHANGES.txt?rev=1152791&r1=1152790&r2=1152791&view=diff
==============================================================================
--- hadoop/common/trunk/common/CHANGES.txt (original)
+++ hadoop/common/trunk/common/CHANGES.txt Mon Aug  1 14:20:24 2011
@@ -296,7 +296,10 @@ Trunk (unreleased changes)
 
     HADOOP-7491. hadoop command should respect HADOOP_OPTS when given
     a class name. (eli)
-    
+
+    HADOOP-7178. Add a parameter, useRawLocalFileSystem, to copyToLocalFile(..)
+    in FileSystem.  (Uma Maheswara Rao G via szetszwo)
+
   OPTIMIZATIONS
   
     HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole

Modified: hadoop/common/trunk/common/src/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/common/src/java/org/apache/hadoop/fs/FileSystem.java?rev=1152791&r1=1152790&r2=1152791&view=diff
==============================================================================
--- hadoop/common/trunk/common/src/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/common/trunk/common/src/java/org/apache/hadoop/fs/FileSystem.java Mon Aug  1 14:20:24 2011
@@ -1707,7 +1707,38 @@ public abstract class FileSystem extends
    */   
   public void copyToLocalFile(boolean delSrc, Path src, Path dst)
     throws IOException {
-    FileUtil.copy(this, src, getLocal(getConf()), dst, delSrc, getConf());
+    copyToLocalFile(delSrc, src, dst, false);
+  }
+  
+    /**
+   * The src file is under FS, and the dst is on the local disk. Copy it from FS
+   * control to the local dst name. delSrc indicates if the src will be removed
+   * or not. useRawLocalFileSystem indicates whether to use RawLocalFileSystem
+   * as local file system or not. RawLocalFileSystem is non crc file system.So,
+   * It will not create any crc files at local.
+   * 
+   * @param delSrc
+   *          whether to delete the src
+   * @param src
+   *          path
+   * @param dst
+   *          path
+   * @param useRawLocalFileSystem
+   *          whether to use RawLocalFileSystem as local file system or not.
+   * 
+   * @throws IOException
+   *           - if any IO error
+   */
+  public void copyToLocalFile(boolean delSrc, Path src, Path dst,
+      boolean useRawLocalFileSystem) throws IOException {
+    Configuration conf = getConf();
+    FileSystem local = null;
+    if (useRawLocalFileSystem) {
+      local = getLocal(conf).getRawFileSystem();
+    } else {
+      local = getLocal(conf);
+    }
+    FileUtil.copy(this, src, local, dst, delSrc, conf);
   }
 
   /**

Modified: hadoop/common/trunk/common/src/test/core/org/apache/hadoop/fs/FSMainOperationsBaseTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/common/src/test/core/org/apache/hadoop/fs/FSMainOperationsBaseTest.java?rev=1152791&r1=1152790&r2=1152791&view=diff
==============================================================================
--- hadoop/common/trunk/common/src/test/core/org/apache/hadoop/fs/FSMainOperationsBaseTest.java (original)
+++ hadoop/common/trunk/common/src/test/core/org/apache/hadoop/fs/FSMainOperationsBaseTest.java Mon Aug  1 14:20:24 2011
@@ -21,9 +21,11 @@ package org.apache.hadoop.fs;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URI;
 
 
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Options.Rename;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.junit.After;
@@ -1080,6 +1082,31 @@ public abstract class FSMainOperationsBa
     Assert.assertNotNull(is);  
   }
   
+  @Test
+  public void testCopyToLocalWithUseRawLocalFileSystemOption() throws Exception {
+    Configuration conf = new Configuration();
+    FileSystem fSys = new RawLocalFileSystem();
+    Path fileToFS = new Path(TEST_ROOT_DIR, "fs.txt");
+    Path fileToLFS = new Path(TEST_ROOT_DIR, "test.txt");
+    Path crcFileAtLFS = new Path(TEST_ROOT_DIR, ".test.txt.crc");
+    fSys.initialize(new URI("file:///"), conf);
+    writeFile(fSys, fileToFS);
+    if (fSys.exists(crcFileAtLFS))
+      Assert.assertTrue("CRC files not deleted", fSys
+          .delete(crcFileAtLFS, true));
+    fSys.copyToLocalFile(false, fileToFS, fileToLFS, true);
+    Assert.assertFalse("CRC files are created", fSys.exists(crcFileAtLFS));
+  }
+
+  private void writeFile(FileSystem fs, Path name) throws IOException {
+    FSDataOutputStream stm = fs.create(name);
+    try {
+      stm.writeBytes("42\n");
+    } finally {
+      stm.close();
+    }
+  }
+  
   protected void createFile(Path path) throws IOException {
     FileSystemTestHelper.createFile(fSys, path);
   }