You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2013/06/25 21:26:48 UTC
svn commit: r1496600 - in
/hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src:
main/java/org/apache/hadoop/mapred/gridmix/
test/java/org/apache/hadoop/mapred/gridmix/
Author: cnauroth
Date: Tue Jun 25 19:26:48 2013
New Revision: 1496600
URL: http://svn.apache.org/r1496600
Log:
HADOOP-9656. Merging change r1496597 from trunk to branch-2.
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java
hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GridmixJob.java
hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/CommonJobTest.java
hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestDistCacheEmulation.java
Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java?rev=1496600&r1=1496599&r2=1496600&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java (original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java Tue Jun 25 19:26:48 2013
@@ -86,6 +86,15 @@ class GenerateDistCacheData extends Grid
"gridmix.distcache.file.list";
static final String JOB_NAME = "GRIDMIX_GENERATE_DISTCACHE_DATA";
+ /**
+ * Create distributed cache file with the permissions 0644.
+ * Since the private distributed cache directory doesn't have execute
+ * permission for others, it is OK to set read permission for others for
+ * the files under that directory and still they will become 'private'
+ * distributed cache files on the simulated cluster.
+ */
+ static final short GRIDMIX_DISTCACHE_FILE_PERM = 0644;
+
public GenerateDistCacheData(Configuration conf) throws IOException {
super(conf, 0L, JOB_NAME);
}
@@ -146,15 +155,8 @@ class GenerateDistCacheData extends Grid
String fileName = new String(value.getBytes(), 0, value.getLength());
Path path = new Path(fileName);
- /**
- * Create distributed cache file with the permissions 0755.
- * Since the private distributed cache directory doesn't have execute
- * permission for others, it is OK to set read permission for others for
- * the files under that directory and still they will become 'private'
- * distributed cache files on the simulated cluster.
- */
FSDataOutputStream dos =
- FileSystem.create(fs, path, new FsPermission((short)0755));
+ FileSystem.create(fs, path, new FsPermission(GRIDMIX_DISTCACHE_FILE_PERM));
int size = 0;
for (long bytes = key.get(); bytes > 0; bytes -= size) {
Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GridmixJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GridmixJob.java?rev=1496600&r1=1496599&r2=1496600&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GridmixJob.java (original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GridmixJob.java Tue Jun 25 19:26:48 2013
@@ -311,7 +311,7 @@ abstract class GridmixJob implements Cal
// set the memory per map task
scaleConfigParameter(sourceConf, destConf,
MRConfig.MAPMEMORY_MB, MRJobConfig.MAP_MEMORY_MB,
- JobConf.DISABLED_MEMORY_LIMIT);
+ MRJobConfig.DEFAULT_MAP_MEMORY_MB);
// validate and fail early
validateTaskMemoryLimits(destConf, MRJobConfig.MAP_MEMORY_MB,
@@ -320,7 +320,7 @@ abstract class GridmixJob implements Cal
// set the memory per reduce task
scaleConfigParameter(sourceConf, destConf,
MRConfig.REDUCEMEMORY_MB, MRJobConfig.REDUCE_MEMORY_MB,
- JobConf.DISABLED_MEMORY_LIMIT);
+ MRJobConfig.DEFAULT_REDUCE_MEMORY_MB);
// validate and fail early
validateTaskMemoryLimits(destConf, MRJobConfig.REDUCE_MEMORY_MB,
JTConfig.JT_MAX_REDUCEMEMORY_MB);
Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/CommonJobTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/CommonJobTest.java?rev=1496600&r1=1496599&r2=1496600&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/CommonJobTest.java (original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/CommonJobTest.java Tue Jun 25 19:26:48 2013
@@ -327,7 +327,8 @@ public class CommonJobTest {
final Path out = GridmixTestUtils.DEST.makeQualified(
GridmixTestUtils.dfs.getUri(),
GridmixTestUtils.dfs.getWorkingDirectory());
- final Path root = new Path(workspace.getAbsolutePath());
+ final Path root = new Path(workspace.getName()).makeQualified(
+ GridmixTestUtils.dfs.getUri(), GridmixTestUtils.dfs.getWorkingDirectory());
if (!workspace.exists()) {
assertTrue(workspace.mkdirs());
}
Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestDistCacheEmulation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestDistCacheEmulation.java?rev=1496600&r1=1496599&r2=1496600&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestDistCacheEmulation.java (original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestDistCacheEmulation.java Tue Jun 25 19:26:48 2013
@@ -134,8 +134,8 @@ public class TestDistCacheEmulation {
FsPermission perm = stat.getPermission();
assertEquals("Wrong permissions for distributed cache file "
- + stat.getPath().toUri().getPath(), new FsPermission((short) 0644),
- perm);
+ + stat.getPath().toUri().getPath(), new FsPermission(
+ GenerateDistCacheData.GRIDMIX_DISTCACHE_FILE_PERM), perm);
}
}