You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2018/03/03 01:01:13 UTC
[25/50] [abbrv] hadoop git commit: MAPREDUCE-7059. Downward
Compatibility issue: MR job fails because of unknown setErasureCodingPolicy
method from 3.x client to HDFS 2.x cluster. Contributed by Jiandan Yang.
MAPREDUCE-7059. Downward Compatibility issue: MR job fails because of unknown setErasureCodingPolicy method from 3.x client to HDFS 2.x cluster. Contributed by Jiandan Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6e6945cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6e6945cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6e6945cd
Branch: refs/heads/HDFS-7240
Commit: 6e6945cd78d76c6beaec85c963f27e28bf96c0f2
Parents: 17f387e
Author: Weiwei Yang <ww...@apache.org>
Authored: Thu Mar 1 10:18:53 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Thu Mar 1 10:18:53 2018 +0800
----------------------------------------------------------------------
.../hadoop/mapreduce/JobResourceUploader.java | 29 +++++++++++++++-----
1 file changed, 22 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e6945cd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
index 03b29bd..1dbecb6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
@@ -37,6 +37,8 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.RpcNoSuchMethodException;
import org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager;
import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -171,7 +173,7 @@ class JobResourceUploader {
if (!conf.getBoolean(MRJobConfig.MR_AM_STAGING_DIR_ERASURECODING_ENABLED,
MRJobConfig.DEFAULT_MR_AM_STAGING_ERASURECODING_ENABLED)) {
- disableErasureCodingForPath(jtFs, submitJobDir);
+ disableErasureCodingForPath(submitJobDir);
}
// Get the resources that have been added via command line arguments in the
@@ -874,13 +876,26 @@ class JobResourceUploader {
return finalPath;
}
- private void disableErasureCodingForPath(FileSystem fs, Path path)
+ private void disableErasureCodingForPath(Path path)
throws IOException {
- if (jtFs instanceof DistributedFileSystem) {
- LOG.info("Disabling Erasure Coding for path: " + path);
- DistributedFileSystem dfs = (DistributedFileSystem) jtFs;
- dfs.setErasureCodingPolicy(path,
- SystemErasureCodingPolicies.getReplicationPolicy().getName());
+ try {
+ if (jtFs instanceof DistributedFileSystem) {
+ LOG.info("Disabling Erasure Coding for path: " + path);
+ DistributedFileSystem dfs = (DistributedFileSystem) jtFs;
+ dfs.setErasureCodingPolicy(path,
+ SystemErasureCodingPolicies.getReplicationPolicy().getName());
+ }
+ } catch (RemoteException e) {
+ if (!RpcNoSuchMethodException.class.getName().equals(e.getClassName())) {
+ throw e;
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Ignore disabling erasure coding for path {} because method "
+ + "disableErasureCodingForPath doesn't exist, probably "
+ + "talking to a lower version HDFS.", path.toString(), e);
+ }
+ }
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org