You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jl...@apache.org on 2015/06/03 22:20:39 UTC
hadoop git commit: MAPREDUCE-6374. Distributed Cache File visibility
should check permission of full path. Contributed by Chang Li
Repository: hadoop
Updated Branches:
refs/heads/trunk e13b671aa -> 107da29ff
MAPREDUCE-6374. Distributed Cache File visibility should check permission of full path. Contributed by Chang Li
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/107da29f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/107da29f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/107da29f
Branch: refs/heads/trunk
Commit: 107da29ff9b6efd02b21c96df60c58ac51590ebd
Parents: e13b671
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Jun 3 20:19:27 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Jun 3 20:19:27 2015 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../ClientDistributedCacheManager.java | 1 +
.../TestClientDistributedCacheManager.java | 28 ++++++++++++++++++++
3 files changed, 32 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107da29f/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 53d3f18..f75b1aa 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -437,6 +437,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6204. TestJobCounters should use new properties instead of
JobConf.MAPRED_TASK_JAVA_OPTS. (Sam Liu via ozawa)
+ MAPREDUCE-6374. Distributed Cache File visibility should check permission
+ of full path (Chang Li via jlowe)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107da29f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java
index 23f3cfc..c15e647 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java
@@ -236,6 +236,7 @@ public class ClientDistributedCacheManager {
Map<URI, FileStatus> statCache) throws IOException {
FileSystem fs = FileSystem.get(uri, conf);
Path current = new Path(uri.getPath());
+ current = fs.makeQualified(current);
//the leaf level file should be readable by others
if (!checkPermissionOfOther(fs, current, FsAction.READ, statCache)) {
return false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/107da29f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/filecache/TestClientDistributedCacheManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/filecache/TestClientDistributedCacheManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/filecache/TestClientDistributedCacheManager.java
index 4824ba3..902cbfc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/filecache/TestClientDistributedCacheManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/filecache/TestClientDistributedCacheManager.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.SequenceFile.CompressionType;
@@ -47,9 +48,13 @@ public class TestClientDistributedCacheManager {
new File(System.getProperty("test.build.data", "/tmp")).toURI()
.toString().replace(' ', '+');
+ private static final String TEST_VISIBILITY_DIR =
+ new File(TEST_ROOT_DIR, "TestCacheVisibility").toURI()
+ .toString().replace(' ', '+');
private FileSystem fs;
private Path firstCacheFile;
private Path secondCacheFile;
+ private Path thirdCacheFile;
private Configuration conf;
@Before
@@ -58,8 +63,10 @@ public class TestClientDistributedCacheManager {
fs = FileSystem.get(conf);
firstCacheFile = new Path(TEST_ROOT_DIR, "firstcachefile");
secondCacheFile = new Path(TEST_ROOT_DIR, "secondcachefile");
+ thirdCacheFile = new Path(TEST_VISIBILITY_DIR,"thirdCachefile");
createTempFile(firstCacheFile, conf);
createTempFile(secondCacheFile, conf);
+ createTempFile(thirdCacheFile, conf);
}
@After
@@ -70,6 +77,9 @@ public class TestClientDistributedCacheManager {
if (!fs.delete(secondCacheFile, false)) {
LOG.warn("Failed to delete secondcachefile");
}
+ if (!fs.delete(thirdCacheFile, false)) {
+ LOG.warn("Failed to delete thirdCachefile");
+ }
}
@Test
@@ -93,6 +103,24 @@ public class TestClientDistributedCacheManager {
Assert.assertEquals(expected, jobConf.get(MRJobConfig.CACHE_FILE_TIMESTAMPS));
}
+ @Test
+ public void testDetermineCacheVisibilities() throws IOException {
+ Path workingdir = new Path(TEST_VISIBILITY_DIR);
+ fs.setWorkingDirectory(workingdir);
+ fs.setPermission(workingdir, new FsPermission((short)00777));
+ fs.setPermission(new Path(TEST_ROOT_DIR), new FsPermission((short)00700));
+ Job job = Job.getInstance(conf);
+ Path relativePath = new Path("thirdCachefile");
+ job.addCacheFile(relativePath.toUri());
+ Configuration jobConf = job.getConfiguration();
+
+ Map<URI, FileStatus> statCache = new HashMap<URI, FileStatus>();
+ ClientDistributedCacheManager.
+ determineCacheVisibilities(jobConf, statCache);
+ Assert.assertFalse(jobConf.
+ getBoolean(MRJobConfig.CACHE_FILE_VISIBILITIES,true));
+ }
+
@SuppressWarnings("deprecation")
void createTempFile(Path p, Configuration conf) throws IOException {
SequenceFile.Writer writer = null;