You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zh...@apache.org on 2015/01/30 22:43:46 UTC
[2/9] hadoop git commit: YARN-3029. FSDownload.unpack() uses local
locale for FS case conversion,
may not work everywhere. Contributed by Varun Saxena.
YARN-3029. FSDownload.unpack() uses local locale for FS case conversion, may not work everywhere. Contributed by Varun Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0501d04
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0501d04
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0501d04
Branch: refs/heads/HDFS-EC
Commit: a0501d04e8bc95dc9dba9263a8440a6ce29fd506
Parents: 0c15190
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Fri Jan 30 04:42:05 2015 +0900
Committer: Zhe Zhang <zh...@apache.org>
Committed: Fri Jan 30 13:42:04 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../org/apache/hadoop/yarn/util/FSDownload.java | 5 +--
.../apache/hadoop/yarn/util/TestFSDownload.java | 33 +++++++++++++++++---
3 files changed, 34 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0501d04/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b582f22..bda4948 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -440,6 +440,9 @@ Release 2.7.0 - UNRELEASED
YARN-3079. Scheduler should also update maximumAllocation when updateNodeResource.
(Zhihai Xu via wangda)
+ YARN-3029. FSDownload.unpack() uses local locale for FS case conversion, may not
+ work everywhere. (Varun Saxena via ozawa)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0501d04/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
index 436cb31..870aa95 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
@@ -23,6 +23,7 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
+import java.util.Locale;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
@@ -271,7 +272,7 @@ public class FSDownload implements Callable<Path> {
private long unpack(File localrsrc, File dst) throws IOException {
switch (resource.getType()) {
case ARCHIVE: {
- String lowerDst = dst.getName().toLowerCase();
+ String lowerDst = dst.getName().toLowerCase(Locale.ENGLISH);
if (lowerDst.endsWith(".jar")) {
RunJar.unJar(localrsrc, dst);
} else if (lowerDst.endsWith(".zip")) {
@@ -290,7 +291,7 @@ public class FSDownload implements Callable<Path> {
}
break;
case PATTERN: {
- String lowerDst = dst.getName().toLowerCase();
+ String lowerDst = dst.getName().toLowerCase(Locale.ENGLISH);
if (lowerDst.endsWith(".jar")) {
String p = resource.getPattern();
RunJar.unJar(localrsrc, dst,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0501d04/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
index 07dd630..3597b31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
@@ -33,6 +33,7 @@ import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.Callable;
@@ -67,7 +68,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.util.Shell;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
@@ -232,7 +232,7 @@ public class TestFSDownload {
byte[] bytes = new byte[len];
r.nextBytes(bytes);
- File archiveFile = new File(p.toUri().getPath() + ".zip");
+ File archiveFile = new File(p.toUri().getPath() + ".ZIP");
archiveFile.createNewFile();
ZipOutputStream out = new ZipOutputStream(
new FileOutputStream(archiveFile));
@@ -243,11 +243,11 @@ public class TestFSDownload {
LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
- + ".zip")));
+ + ".ZIP")));
ret.setSize(len);
ret.setType(LocalResourceType.ARCHIVE);
ret.setVisibility(vis);
- ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".zip"))
+ ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".ZIP"))
.getModificationTime());
return ret;
}
@@ -474,8 +474,8 @@ public class TestFSDownload {
int size = rand.nextInt(512) + 512;
LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
-
Path p = new Path(basedir, "" + 1);
+ String strFileName = "";
LocalResource rsrc = null;
switch (fileType) {
case TAR:
@@ -487,6 +487,7 @@ public class TestFSDownload {
break;
case ZIP:
rsrc = createZipFile(files, p, size, rand, vis);
+ strFileName = p.getName() + ".ZIP";
break;
case TGZ:
rsrc = createTgzFile(files, p, size, rand, vis);
@@ -509,6 +510,13 @@ public class TestFSDownload {
FileStatus[] childFiles = files.getDefaultFileSystem().listStatus(
filestatus.getPath());
for (FileStatus childfile : childFiles) {
+ if(strFileName.endsWith(".ZIP") &&
+ childfile.getPath().getName().equals(strFileName) &&
+ !childfile.isDirectory()) {
+ Assert.fail("Failure...After unzip, there should have been a" +
+ " directory formed with zip file name but found a file. "
+ + childfile.getPath());
+ }
if (childfile.getPath().getName().startsWith("tmp")) {
Assert.fail("Tmp File should not have been there "
+ childfile.getPath());
@@ -539,6 +547,21 @@ public class TestFSDownload {
downloadWithFileType(TEST_FILE_TYPE.ZIP);
}
+ /*
+ * To test fix for YARN-3029
+ */
+ @Test (timeout=10000)
+ public void testDownloadArchiveZipWithTurkishLocale() throws IOException,
+ URISyntaxException, InterruptedException {
+ Locale defaultLocale = Locale.getDefault();
+ // Set to Turkish
+ Locale turkishLocale = new Locale("tr", "TR");
+ Locale.setDefault(turkishLocale);
+ downloadWithFileType(TEST_FILE_TYPE.ZIP);
+ // Set the locale back to original default locale
+ Locale.setDefault(defaultLocale);
+ }
+
@Test (timeout=10000)
public void testDownloadArchiveTgz() throws IOException, URISyntaxException,
InterruptedException {