You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2013/06/19 19:39:31 UTC
svn commit: r1494705 - in
/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./
src/main/java/org/apache/hadoop/fs/ src/test/java/org/apache/hadoop/fs/
Author: cnauroth
Date: Wed Jun 19 17:39:30 2013
New Revision: 1494705
URL: http://svn.apache.org/r1494705
Log:
HADOOP-9264. Merging change r1441172 from trunk to branch-2.
Added:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tar
- copied unchanged from r1441172, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tar
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tgz
- copied unchanged from r1441172, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tgz
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1494705&r1=1494704&r2=1494705&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Wed Jun 19 17:39:30 2013
@@ -467,6 +467,9 @@ Release 2.1.0-beta - UNRELEASED
HADOOP-9637. Adding Native Fstat for Windows as needed by YARN. (Chuan Liu
via cnauroth)
+ HADOOP-9264. Port change to use Java untar API on Windows from
+ branch-1-win to trunk. (Chris Nauroth via suresh)
+
Release 2.0.5-alpha - 06/06/2013
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml?rev=1494705&r1=1494704&r2=1494705&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml Wed Jun 19 17:39:30 2013
@@ -249,6 +249,11 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ <version>1.4</version>
+ </dependency>
</dependencies>
<build>
@@ -396,6 +401,23 @@
</configuration>
</execution>
<execution>
+ <id>copy-test-tarballs</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <copy toDir="${test.cache.data}">
+ <fileset dir="${basedir}/src/test/java/org/apache/hadoop/fs">
+ <include name="test-untar.tar"/>
+ <include name="test-untar.tgz"/>
+ </fileset>
+ </copy>
+ </target>
+ </configuration>
+ </execution>
+ <execution>
<phase>pre-site</phase>
<goals>
<goal>run</goal>
@@ -426,6 +448,7 @@
<exclude>src/test/all-tests</exclude>
<exclude>src/test/resources/kdc/ldif/users.ldif</exclude>
<exclude>src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.c</exclude>
+ <exclude>src/test/java/org/apache/hadoop/fs/test-untar.tgz</exclude>
</excludes>
</configuration>
</plugin>
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java?rev=1494705&r1=1494704&r2=1494705&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java Wed Jun 19 17:39:30 2013
@@ -27,10 +27,13 @@ import java.util.Map;
import java.util.jar.Attributes;
import java.util.jar.JarOutputStream;
import java.util.jar.Manifest;
+import java.util.zip.GZIPInputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import org.apache.commons.collections.map.CaseInsensitiveMap;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -609,14 +612,28 @@ public class FileUtil {
* @throws IOException
*/
public static void unTar(File inFile, File untarDir) throws IOException {
- if (!untarDir.mkdirs()) {
+ if (!untarDir.mkdirs()) {
if (!untarDir.isDirectory()) {
throw new IOException("Mkdirs failed to create " + untarDir);
}
}
- StringBuilder untarCommand = new StringBuilder();
boolean gzipped = inFile.toString().endsWith("gz");
+ if(Shell.WINDOWS) {
+ // Tar is not native to Windows. Use simple Java based implementation for
+ // tests and simple tar archives
+ unTarUsingJava(inFile, untarDir, gzipped);
+ }
+ else {
+ // spawn tar utility to untar archive for full fledged unix behavior such
+ // as resolving symlinks in tar archives
+ unTarUsingTar(inFile, untarDir, gzipped);
+ }
+ }
+
+ private static void unTarUsingTar(File inFile, File untarDir,
+ boolean gzipped) throws IOException {
+ StringBuffer untarCommand = new StringBuffer();
if (gzipped) {
untarCommand.append(" gzip -dc '");
untarCommand.append(FileUtil.makeShellPath(inFile));
@@ -641,7 +658,62 @@ public class FileUtil {
". Tar process exited with exit code " + exitcode);
}
}
+
+ private static void unTarUsingJava(File inFile, File untarDir,
+ boolean gzipped) throws IOException {
+ InputStream inputStream = null;
+ if (gzipped) {
+ inputStream = new BufferedInputStream(new GZIPInputStream(
+ new FileInputStream(inFile)));
+ } else {
+ inputStream = new BufferedInputStream(new FileInputStream(inFile));
+ }
+
+ TarArchiveInputStream tis = new TarArchiveInputStream(inputStream);
+
+ for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null;) {
+ unpackEntries(tis, entry, untarDir);
+ entry = tis.getNextTarEntry();
+ }
+ }
+
+ private static void unpackEntries(TarArchiveInputStream tis,
+ TarArchiveEntry entry, File outputDir) throws IOException {
+ if (entry.isDirectory()) {
+ File subDir = new File(outputDir, entry.getName());
+ if (!subDir.mkdir() && !subDir.isDirectory()) {
+ throw new IOException("Mkdirs failed to create tar internal dir "
+ + outputDir);
+ }
+ for (TarArchiveEntry e : entry.getDirectoryEntries()) {
+ unpackEntries(tis, e, subDir);
+ }
+
+ return;
+ }
+
+ File outputFile = new File(outputDir, entry.getName());
+ if (!outputDir.exists()) {
+ if (!outputDir.mkdirs()) {
+ throw new IOException("Mkdirs failed to create tar internal dir "
+ + outputDir);
+ }
+ }
+
+ int count;
+ byte data[] = new byte[2048];
+ BufferedOutputStream outputStream = new BufferedOutputStream(
+ new FileOutputStream(outputFile));
+
+ while ((count = tis.read(data)) != -1) {
+ outputStream.write(data, 0, count);
+ }
+
+ outputStream.flush();
+ outputStream.close();
+ }
+
/**
* Class for creating hardlinks.
* Supports Unix, WindXP.
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java?rev=1494705&r1=1494704&r2=1494705&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java Wed Jun 19 17:39:30 2013
@@ -564,6 +564,176 @@ public class TestFileUtil {
}
@Test (timeout = 30000)
+ public void testSymlink() throws Exception {
+ Assert.assertFalse(del.exists());
+ del.mkdirs();
+
+ byte[] data = "testSymLink".getBytes();
+
+ File file = new File(del, FILE);
+ File link = new File(del, "_link");
+
+ //write some data to the file
+ FileOutputStream os = new FileOutputStream(file);
+ os.write(data);
+ os.close();
+
+ //create the symlink
+ FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath());
+
+ //ensure that symlink length is correctly reported by Java
+ Assert.assertEquals(data.length, file.length());
+ Assert.assertEquals(data.length, link.length());
+
+ //ensure that we can read from link.
+ FileInputStream in = new FileInputStream(link);
+ long len = 0;
+ while (in.read() > 0) {
+ len++;
+ }
+ in.close();
+ Assert.assertEquals(data.length, len);
+ }
+
+ /**
+ * Test that rename on a symlink works as expected.
+ */
+ @Test (timeout = 30000)
+ public void testSymlinkRenameTo() throws Exception {
+ Assert.assertFalse(del.exists());
+ del.mkdirs();
+
+ File file = new File(del, FILE);
+ file.createNewFile();
+ File link = new File(del, "_link");
+
+ // create the symlink
+ FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath());
+
+ Assert.assertTrue(file.exists());
+ Assert.assertTrue(link.exists());
+
+ File link2 = new File(del, "_link2");
+
+ // Rename the symlink
+ Assert.assertTrue(link.renameTo(link2));
+
+ // Make sure the file still exists
+ // (NOTE: this would fail on Java6 on Windows if we didn't
+ // copy the file in FileUtil#symlink)
+ Assert.assertTrue(file.exists());
+
+ Assert.assertTrue(link2.exists());
+ Assert.assertFalse(link.exists());
+ }
+
+ /**
+ * Test that deletion of a symlink works as expected.
+ */
+ @Test (timeout = 30000)
+ public void testSymlinkDelete() throws Exception {
+ Assert.assertFalse(del.exists());
+ del.mkdirs();
+
+ File file = new File(del, FILE);
+ file.createNewFile();
+ File link = new File(del, "_link");
+
+ // create the symlink
+ FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath());
+
+ Assert.assertTrue(file.exists());
+ Assert.assertTrue(link.exists());
+
+ // make sure that deleting a symlink works properly
+ Assert.assertTrue(link.delete());
+ Assert.assertFalse(link.exists());
+ Assert.assertTrue(file.exists());
+ }
+
+ /**
+ * Test that length on a symlink works as expected.
+ */
+ @Test (timeout = 30000)
+ public void testSymlinkLength() throws Exception {
+ Assert.assertFalse(del.exists());
+ del.mkdirs();
+
+ byte[] data = "testSymLinkData".getBytes();
+
+ File file = new File(del, FILE);
+ File link = new File(del, "_link");
+
+ // write some data to the file
+ FileOutputStream os = new FileOutputStream(file);
+ os.write(data);
+ os.close();
+
+ Assert.assertEquals(0, link.length());
+
+ // create the symlink
+ FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath());
+
+ // ensure that File#length returns the target file and link size
+ Assert.assertEquals(data.length, file.length());
+ Assert.assertEquals(data.length, link.length());
+
+ file.delete();
+ Assert.assertFalse(file.exists());
+
+ if (Shell.WINDOWS && !Shell.isJava7OrAbove()) {
+ // On Java6 on Windows, we copied the file
+ Assert.assertEquals(data.length, link.length());
+ } else {
+ // Otherwise, the target file size is zero
+ Assert.assertEquals(0, link.length());
+ }
+
+ link.delete();
+ Assert.assertFalse(link.exists());
+ }
+
+ private void doUntarAndVerify(File tarFile, File untarDir)
+ throws IOException {
+ if (untarDir.exists() && !FileUtil.fullyDelete(untarDir)) {
+ throw new IOException("Could not delete directory '" + untarDir + "'");
+ }
+ FileUtil.unTar(tarFile, untarDir);
+
+ String parentDir = untarDir.getCanonicalPath() + Path.SEPARATOR + "name";
+ File testFile = new File(parentDir + Path.SEPARATOR + "version");
+ Assert.assertTrue(testFile.exists());
+ Assert.assertTrue(testFile.length() == 0);
+ String imageDir = parentDir + Path.SEPARATOR + "image";
+ testFile = new File(imageDir + Path.SEPARATOR + "fsimage");
+ Assert.assertTrue(testFile.exists());
+ Assert.assertTrue(testFile.length() == 157);
+ String currentDir = parentDir + Path.SEPARATOR + "current";
+ testFile = new File(currentDir + Path.SEPARATOR + "fsimage");
+ Assert.assertTrue(testFile.exists());
+ Assert.assertTrue(testFile.length() == 4331);
+ testFile = new File(currentDir + Path.SEPARATOR + "edits");
+ Assert.assertTrue(testFile.exists());
+ Assert.assertTrue(testFile.length() == 1033);
+ testFile = new File(currentDir + Path.SEPARATOR + "fstime");
+ Assert.assertTrue(testFile.exists());
+ Assert.assertTrue(testFile.length() == 8);
+ }
+
+ @Test (timeout = 30000)
+ public void testUntar() throws IOException {
+ String tarGzFileName = System.getProperty("test.cache.data",
+ "build/test/cache") + "/test-untar.tgz";
+ String tarFileName = System.getProperty("test.cache.data",
+ "build/test/cache") + "/test-untar.tar";
+ String dataDir = System.getProperty("test.build.data", "build/test/data");
+ File untarDir = new File(dataDir, "untarDir");
+
+ doUntarAndVerify(new File(tarGzFileName), untarDir);
+ doUntarAndVerify(new File(tarFileName), untarDir);
+ }
+
+ @Test (timeout = 30000)
public void testCreateJarWithClassPath() throws Exception {
// setup test directory for files
Assert.assertFalse(tmp.exists());