You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2013/06/04 01:42:28 UTC
svn commit: r1489227 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/
hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/o...
Author: acmurthy
Date: Mon Jun 3 23:42:28 2013
New Revision: 1489227
URL: http://svn.apache.org/r1489227
Log:
MAPREDUCE-5300. Fix backward incompatibility for o.a.h.mapreduce.filecache.DistributedCache. Contributed by Zhijie Shen.
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/JobContextImpl.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1489227&r1=1489226&r2=1489227&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Mon Jun 3 23:42:28 2013
@@ -487,6 +487,9 @@ Release 2.1.0-beta - UNRELEASED
MAPREDUCE-5296. Fix backward incompatibility for JobControl by adding the
omitted addJob. (Zhijie Shen via acmurthy)
+ MAPREDUCE-5300. Fix backward incompatibility for
+ o.a.h.mapreduce.filecache.DistributedCache. (Zhijie Shen via acmurthy)
+
BREAKDOWN OF HADOOP-8562 SUBTASKS
MAPREDUCE-4739. Some MapReduce tests fail to find winutils.
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java?rev=1489227&r1=1489226&r2=1489227&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java Mon Jun 3 23:42:28 2013
@@ -336,17 +336,6 @@ public class MRApps extends Apps {
return startCommitFile;
}
- private static long[] parseTimeStamps(String[] strs) {
- if (null == strs) {
- return null;
- }
- long[] result = new long[strs.length];
- for(int i=0; i < strs.length; ++i) {
- result[i] = Long.parseLong(strs[i]);
- }
- return result;
- }
-
public static void setupDistributedCache(
Configuration conf,
Map<String, LocalResource> localResources)
@@ -356,7 +345,7 @@ public class MRApps extends Apps {
parseDistributedCacheArtifacts(conf, localResources,
LocalResourceType.ARCHIVE,
DistributedCache.getCacheArchives(conf),
- parseTimeStamps(DistributedCache.getArchiveTimestamps(conf)),
+ DistributedCache.getArchiveTimestamps(conf),
getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES),
DistributedCache.getArchiveVisibilities(conf));
@@ -365,7 +354,7 @@ public class MRApps extends Apps {
localResources,
LocalResourceType.FILE,
DistributedCache.getCacheFiles(conf),
- parseTimeStamps(DistributedCache.getFileTimestamps(conf)),
+ DistributedCache.getFileTimestamps(conf),
getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES),
DistributedCache.getFileVisibilities(conf));
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java?rev=1489227&r1=1489226&r2=1489227&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java Mon Jun 3 23:42:28 2013
@@ -245,14 +245,14 @@ public class TestMRWithDistributedCache
Assert.assertEquals(1,
DistributedCache.getArchiveTimestamps(conf).length);
Assert.assertEquals(1234567890,
- Long.parseLong(DistributedCache.getArchiveTimestamps(conf)[0]));
+ DistributedCache.getArchiveTimestamps(conf)[0]);
DistributedCache.setFileTimestamps(conf, "1234567890");
Assert.assertEquals(1234567890,
conf.getLong(DistributedCache.CACHE_FILES_TIMESTAMPS, 0));
Assert.assertEquals(1,
DistributedCache.getFileTimestamps(conf).length);
Assert.assertEquals(1234567890,
- Long.parseLong(DistributedCache.getFileTimestamps(conf)[0]));
+ DistributedCache.getFileTimestamps(conf)[0]);
DistributedCache.createAllSymlink(conf, new File("Test Job Cache Dir"),
new File("Test Work Dir"));
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java?rev=1489227&r1=1489226&r2=1489227&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java Mon Jun 3 23:42:28 2013
@@ -215,16 +215,33 @@ public class DistributedCache {
}
/**
+ * Parse a list of strings into longs.
+ * @param strs the list of strings to parse
+ * @return a list of longs that were parsed. same length as strs.
+ */
+ private static long[] parseTimestamps(String[] strs) {
+ if (strs == null) {
+ return null;
+ }
+ long[] result = new long[strs.length];
+ for(int i=0; i < strs.length; ++i) {
+ result[i] = Long.parseLong(strs[i]);
+ }
+ return result;
+ }
+
+ /**
* Get the timestamps of the archives. Used by internal
* DistributedCache and MapReduce code.
* @param conf The configuration which stored the timestamps
- * @return a string array of timestamps
+ * @return a long array of timestamps
* @throws IOException
* @deprecated Use {@link JobContext#getArchiveTimestamps()} instead
*/
@Deprecated
- public static String[] getArchiveTimestamps(Configuration conf) {
- return conf.getStrings(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS);
+ public static long[] getArchiveTimestamps(Configuration conf) {
+ return parseTimestamps(
+ conf.getStrings(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS));
}
@@ -232,13 +249,14 @@ public class DistributedCache {
* Get the timestamps of the files. Used by internal
* DistributedCache and MapReduce code.
* @param conf The configuration which stored the timestamps
- * @return a string array of timestamps
+ * @return a long array of timestamps
* @throws IOException
* @deprecated Use {@link JobContext#getFileTimestamps()} instead
*/
@Deprecated
- public static String[] getFileTimestamps(Configuration conf) {
- return conf.getStrings(MRJobConfig.CACHE_FILE_TIMESTAMPS);
+ public static long[] getFileTimestamps(Configuration conf) {
+ return parseTimestamps(
+ conf.getStrings(MRJobConfig.CACHE_FILE_TIMESTAMPS));
}
/**
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/JobContextImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/JobContextImpl.java?rev=1489227&r1=1489226&r2=1489227&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/JobContextImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/JobContextImpl.java Mon Jun 3 23:42:28 2013
@@ -342,7 +342,23 @@ public class JobContextImpl implements J
public Path[] getFileClassPaths() {
return DistributedCache.getFileClassPaths(conf);
}
-
+
+ /**
+ * Parse a list of longs into strings.
+ * @param timestamps the list of longs to parse
+ * @return a list of string that were parsed. same length as timestamps.
+ */
+ private static String[] toTimestampStrs(long[] timestamps) {
+ if (timestamps == null) {
+ return null;
+ }
+ String[] result = new String[timestamps.length];
+ for(int i=0; i < timestamps.length; ++i) {
+ result[i] = Long.toString(timestamps[i]);
+ }
+ return result;
+ }
+
/**
* Get the timestamps of the archives. Used by internal
* DistributedCache and MapReduce code.
@@ -350,7 +366,7 @@ public class JobContextImpl implements J
* @throws IOException
*/
public String[] getArchiveTimestamps() {
- return DistributedCache.getArchiveTimestamps(conf);
+ return toTimestampStrs(DistributedCache.getArchiveTimestamps(conf));
}
/**
@@ -360,7 +376,7 @@ public class JobContextImpl implements J
* @throws IOException
*/
public String[] getFileTimestamps() {
- return DistributedCache.getFileTimestamps(conf);
+ return toTimestampStrs(DistributedCache.getFileTimestamps(conf));
}
/**