You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by cn...@apache.org on 2013/06/19 20:08:44 UTC
svn commit: r1494716 - in
/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project: CHANGES.txt
hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
Author: cnauroth
Date: Wed Jun 19 18:08:43 2013
New Revision: 1494716
URL: http://svn.apache.org/r1494716
Log:
YARN-597. Merging change r1494713 from branch-2 to branch-2.1-beta.
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt?rev=1494716&r1=1494715&r2=1494716&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt Wed Jun 19 18:08:43 2013
@@ -633,6 +633,9 @@ Release 2.1.0-beta - UNRELEASED
YARN-839. TestContainerLaunch.testContainerEnvVariables fails on Windows.
(Chuan Liu via cnauroth)
+ YARN-597. TestFSDownload fails on Windows due to dependencies on
+ tar/gzip/jar tools. (Ivan Mitic via acmurthy)
+
YARN-795. Fair scheduler queue metrics should subtract allocated vCores from
available vCores. (ywskycn via tucu)
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java?rev=1494716&r1=1494715&r2=1494716&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java Wed Jun 19 18:08:43 2013
@@ -25,7 +25,6 @@ import static org.junit.Assert.assertEqu
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.net.URI;
import java.net.URISyntaxException;
import java.util.EnumSet;
import java.util.HashMap;
@@ -37,11 +36,16 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.jar.JarEntry;
import java.util.jar.JarOutputStream;
import java.util.jar.Manifest;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
import junit.framework.Assert;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -50,12 +54,10 @@ import org.apache.hadoop.fs.FSDataOutput
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@@ -123,32 +125,20 @@ public class TestFSDownload {
static LocalResource createTarFile(FileContext files, Path p, int len,
Random r, LocalResourceVisibility vis) throws IOException,
URISyntaxException {
+ byte[] bytes = new byte[len];
+ r.nextBytes(bytes);
+
+ File archiveFile = new File(p.toUri().getPath() + ".tar");
+ archiveFile.createNewFile();
+ TarArchiveOutputStream out = new TarArchiveOutputStream(
+ new FileOutputStream(archiveFile));
+ TarArchiveEntry entry = new TarArchiveEntry(p.getName());
+ entry.setSize(bytes.length);
+ out.putArchiveEntry(entry);
+ out.write(bytes);
+ out.closeArchiveEntry();
+ out.close();
- FSDataOutputStream outFile = null;
- try {
- byte[] bytes = new byte[len];
- Path tarPath = new Path(p.toString());
- outFile = files.create(tarPath, EnumSet.of(CREATE, OVERWRITE));
- r.nextBytes(bytes);
- outFile.write(bytes);
- } finally {
- if (outFile != null)
- outFile.close();
- }
- StringBuffer tarCommand = new StringBuffer();
- URI u = new URI(p.getParent().toString());
- tarCommand.append("cd '");
- tarCommand.append(FileUtil.makeShellPath(u.getPath().toString()));
- tarCommand.append("' ; ");
- tarCommand.append("tar -czf " + p.getName() + ".tar " + p.getName());
- String[] shellCmd = { "bash", "-c", tarCommand.toString() };
- ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
- shexec.execute();
- int exitcode = shexec.getExitCode();
- if (exitcode != 0) {
- throw new IOException("Error untarring file " + p
- + ". Tar process exited with exit code " + exitcode);
- }
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
+ ".tar")));
@@ -163,32 +153,18 @@ public class TestFSDownload {
static LocalResource createJarFile(FileContext files, Path p, int len,
Random r, LocalResourceVisibility vis) throws IOException,
URISyntaxException {
+ byte[] bytes = new byte[len];
+ r.nextBytes(bytes);
+
+ File archiveFile = new File(p.toUri().getPath() + ".jar");
+ archiveFile.createNewFile();
+ JarOutputStream out = new JarOutputStream(
+ new FileOutputStream(archiveFile));
+ out.putNextEntry(new JarEntry(p.getName()));
+ out.write(bytes);
+ out.closeEntry();
+ out.close();
- FSDataOutputStream outFile = null;
- try {
- byte[] bytes = new byte[len];
- Path tarPath = new Path(p.toString());
- outFile = files.create(tarPath, EnumSet.of(CREATE, OVERWRITE));
- r.nextBytes(bytes);
- outFile.write(bytes);
- } finally {
- if (outFile != null)
- outFile.close();
- }
- StringBuffer tarCommand = new StringBuffer();
- URI u = new URI(p.getParent().toString());
- tarCommand.append("cd '");
- tarCommand.append(FileUtil.makeShellPath(u.getPath().toString()));
- tarCommand.append("' ; ");
- tarCommand.append("jar cf " + p.getName() + ".jar " + p.getName());
- String[] shellCmd = { "bash", "-c", tarCommand.toString() };
- ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
- shexec.execute();
- int exitcode = shexec.getExitCode();
- if (exitcode != 0) {
- throw new IOException("Error untarring file " + p
- + ". Tar process exited with exit code " + exitcode);
- }
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
+ ".jar")));
@@ -203,39 +179,25 @@ public class TestFSDownload {
static LocalResource createZipFile(FileContext files, Path p, int len,
Random r, LocalResourceVisibility vis) throws IOException,
URISyntaxException {
+ byte[] bytes = new byte[len];
+ r.nextBytes(bytes);
+
+ File archiveFile = new File(p.toUri().getPath() + ".zip");
+ archiveFile.createNewFile();
+ ZipOutputStream out = new ZipOutputStream(
+ new FileOutputStream(archiveFile));
+ out.putNextEntry(new ZipEntry(p.getName()));
+ out.write(bytes);
+ out.closeEntry();
+ out.close();
- FSDataOutputStream outFile = null;
- try {
- byte[] bytes = new byte[len];
- Path tarPath = new Path(p.toString());
- outFile = files.create(tarPath, EnumSet.of(CREATE, OVERWRITE));
- r.nextBytes(bytes);
- outFile.write(bytes);
- } finally {
- if (outFile != null)
- outFile.close();
- }
- StringBuffer zipCommand = new StringBuffer();
- URI u = new URI(p.getParent().toString());
- zipCommand.append("cd '");
- zipCommand.append(FileUtil.makeShellPath(u.getPath().toString()));
- zipCommand.append("' ; ");
- zipCommand.append("gzip " + p.getName());
- String[] shellCmd = { "bash", "-c", zipCommand.toString() };
- ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd);
- shexec.execute();
- int exitcode = shexec.getExitCode();
- if (exitcode != 0) {
- throw new IOException("Error untarring file " + p
- + ". Tar process exited with exit code " + exitcode);
- }
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
+ ".zip")));
ret.setSize(len);
ret.setType(LocalResourceType.ARCHIVE);
ret.setVisibility(vis);
- ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".gz"))
+ ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".zip"))
.getModificationTime());
return ret;
}