You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ba...@apache.org on 2016/07/21 18:44:19 UTC
falcon git commit: FALCON-2090 HDFS Snapshot failed with
UnknownHostException in HA mode
Repository: falcon
Updated Branches:
refs/heads/master 752511bbe -> 3ed804e92
FALCON-2090 HDFS Snapshot failed with UnknownHostException in HA mode
\u2026uling in HA Mode
Author: bvellanki <bv...@hortonworks.com>
Reviewers: "Peeyush B <pb...@hortonworks.com>"
Closes #237 from bvellanki/FALCON-2090
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/3ed804e9
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/3ed804e9
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/3ed804e9
Branch: refs/heads/master
Commit: 3ed804e92d1434e42a057dc68a6faa6d2fb248ce
Parents: 752511b
Author: bvellanki <bv...@hortonworks.com>
Authored: Thu Jul 21 11:44:14 2016 -0700
Committer: bvellanki <bv...@hortonworks.com>
Committed: Thu Jul 21 11:44:14 2016 -0700
----------------------------------------------------------------------
.../replication/HdfsSnapshotReplicator.java | 8 ++++++--
.../snapshots/retention/HdfsSnapshotEvictor.java | 6 ++++--
.../falcon/snapshots/util/HdfsSnapshotUtil.java | 15 ++++++++++-----
.../replication/HdfsSnapshotReplicatorTest.java | 12 ++++++------
.../java/org/apache/falcon/entity/ClusterHelper.java | 5 ++---
5 files changed, 28 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/3ed804e9/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicator.java
----------------------------------------------------------------------
diff --git a/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicator.java b/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicator.java
index 2e41cc0..6f5defe 100644
--- a/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicator.java
+++ b/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicator.java
@@ -73,8 +73,12 @@ public class HdfsSnapshotReplicator extends Configured implements Tool {
String sourceStorageUrl = cmd.getOptionValue(HdfsSnapshotMirrorProperties.SOURCE_NN.getName());
String targetStorageUrl = cmd.getOptionValue(HdfsSnapshotMirrorProperties.TARGET_NN.getName());
- DistributedFileSystem sourceFs = HdfsSnapshotUtil.getSourceFileSystem(cmd);
- DistributedFileSystem targetFs = HdfsSnapshotUtil.getTargetFileSystem(cmd);
+ // Always add to getConf() so that configuration set by oozie action is
+ // available when creating DistributedFileSystem.
+ DistributedFileSystem sourceFs = HdfsSnapshotUtil.getSourceFileSystem(cmd,
+ new Configuration(getConf()));
+ DistributedFileSystem targetFs = HdfsSnapshotUtil.getTargetFileSystem(cmd,
+ new Configuration(getConf()));
String currentSnapshotName = HdfsSnapshotUtil.SNAPSHOT_PREFIX
+ cmd.getOptionValue(HdfsSnapshotMirrorProperties.SNAPSHOT_JOB_NAME.getName())
http://git-wip-us.apache.org/repos/asf/falcon/blob/3ed804e9/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/retention/HdfsSnapshotEvictor.java
----------------------------------------------------------------------
diff --git a/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/retention/HdfsSnapshotEvictor.java b/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/retention/HdfsSnapshotEvictor.java
index 22e3377..a50e770 100644
--- a/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/retention/HdfsSnapshotEvictor.java
+++ b/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/retention/HdfsSnapshotEvictor.java
@@ -60,8 +60,10 @@ public class HdfsSnapshotEvictor extends Configured implements Tool {
@Override
public int run(String[] args) throws Exception {
CommandLine cmd = getCommand(args);
- DistributedFileSystem sourceFs = HdfsSnapshotUtil.getSourceFileSystem(cmd);
- DistributedFileSystem targetFs = HdfsSnapshotUtil.getTargetFileSystem(cmd);
+ DistributedFileSystem sourceFs = HdfsSnapshotUtil.getSourceFileSystem(cmd,
+ new Configuration(getConf()));
+ DistributedFileSystem targetFs = HdfsSnapshotUtil.getTargetFileSystem(cmd,
+ new Configuration(getConf()));
String sourceDir = cmd.getOptionValue(HdfsSnapshotMirrorProperties.SOURCE_SNAPSHOT_DIR.getName());
String targetDir = cmd.getOptionValue(HdfsSnapshotMirrorProperties.TARGET_SNAPSHOT_DIR.getName());
http://git-wip-us.apache.org/repos/asf/falcon/blob/3ed804e9/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/util/HdfsSnapshotUtil.java
----------------------------------------------------------------------
diff --git a/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/util/HdfsSnapshotUtil.java b/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/util/HdfsSnapshotUtil.java
index 5196791..88f6fd9 100644
--- a/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/util/HdfsSnapshotUtil.java
+++ b/addons/hdfs-snapshot-mirroring/src/main/java/org/apache/falcon/snapshots/util/HdfsSnapshotUtil.java
@@ -19,6 +19,7 @@
package org.apache.falcon.snapshots.util;
import org.apache.commons.cli.CommandLine;
+import org.apache.commons.lang3.StringUtils;
import org.apache.falcon.FalconException;
import org.apache.falcon.entity.ClusterHelper;
import org.apache.falcon.extensions.mirroring.hdfsSnapshot.HdfsSnapshotMirrorProperties;
@@ -37,29 +38,33 @@ public final class HdfsSnapshotUtil {
private HdfsSnapshotUtil() {}
- public static DistributedFileSystem getSourceFileSystem(CommandLine cmd) throws FalconException {
+ public static DistributedFileSystem getSourceFileSystem(CommandLine cmd,
+ Configuration conf) throws FalconException {
String sourceStorageUrl = cmd.getOptionValue(HdfsSnapshotMirrorProperties.SOURCE_NN.getName());
String sourceExecuteEndpoint = cmd.getOptionValue(HdfsSnapshotMirrorProperties.SOURCE_EXEC_URL.getName());
String sourcePrincipal = parseKerberosPrincipal(cmd.getOptionValue(
HdfsSnapshotMirrorProperties.SOURCE_NN_KERBEROS_PRINCIPAL.getName()));
- Configuration sourceConf = ClusterHelper.getConfiguration(sourceStorageUrl,
+
+ Configuration sourceConf = ClusterHelper.getConfiguration(conf, sourceStorageUrl,
sourceExecuteEndpoint, sourcePrincipal);
return HadoopClientFactory.get().createDistributedProxiedFileSystem(sourceConf);
}
- public static DistributedFileSystem getTargetFileSystem(CommandLine cmd) throws FalconException {
+ public static DistributedFileSystem getTargetFileSystem(CommandLine cmd,
+ Configuration conf) throws FalconException {
String targetStorageUrl = cmd.getOptionValue(HdfsSnapshotMirrorProperties.TARGET_NN.getName());
String taregtExecuteEndpoint = cmd.getOptionValue(HdfsSnapshotMirrorProperties.TARGET_EXEC_URL.getName());
String targetPrincipal = parseKerberosPrincipal(cmd.getOptionValue(
HdfsSnapshotMirrorProperties.TARGET_NN_KERBEROS_PRINCIPAL.getName()));
- Configuration targetConf = ClusterHelper.getConfiguration(targetStorageUrl,
+ Configuration targetConf = ClusterHelper.getConfiguration(conf, targetStorageUrl,
taregtExecuteEndpoint, targetPrincipal);
return HadoopClientFactory.get().createDistributedProxiedFileSystem(targetConf);
}
public static String parseKerberosPrincipal(String principal) {
- if (principal.equals(HdfsSnapshotMirroringExtension.EMPTY_KERBEROS_PRINCIPAL)) {
+ if (StringUtils.isEmpty(principal)
+ || principal.equals(HdfsSnapshotMirroringExtension.EMPTY_KERBEROS_PRINCIPAL)) {
return null;
}
return principal;
http://git-wip-us.apache.org/repos/asf/falcon/blob/3ed804e9/addons/hdfs-snapshot-mirroring/src/test/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicatorTest.java
----------------------------------------------------------------------
diff --git a/addons/hdfs-snapshot-mirroring/src/test/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicatorTest.java b/addons/hdfs-snapshot-mirroring/src/test/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicatorTest.java
index 7924214..fe7ced5 100644
--- a/addons/hdfs-snapshot-mirroring/src/test/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicatorTest.java
+++ b/addons/hdfs-snapshot-mirroring/src/test/java/org/apache/falcon/snapshots/replication/HdfsSnapshotReplicatorTest.java
@@ -25,7 +25,7 @@ import org.apache.falcon.entity.store.ConfigurationStore;
import org.apache.falcon.entity.v0.EntityType;
import org.apache.falcon.entity.v0.cluster.Cluster;
import org.apache.falcon.extensions.mirroring.hdfsSnapshot.HdfsSnapshotMirrorProperties;
-import org.apache.falcon.hadoop.HadoopClientFactory;
+import org.apache.falcon.snapshots.util.HdfsSnapshotUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
@@ -72,6 +72,7 @@ public class HdfsSnapshotReplicatorTest extends HdfsSnapshotReplicator {
@BeforeClass
public void init() throws Exception {
+ this.setConf(new Configuration());
baseDir = Files.createTempDirectory("test_snapshot-replication").toFile().getAbsoluteFile();
miniDFSCluster = MiniHdfsClusterUtil.initMiniDfs(MiniHdfsClusterUtil.SNAPSHOT_REPL_TEST_PORT, baseDir);
miniDfs = miniDFSCluster.getFileSystem();
@@ -100,14 +101,13 @@ public class HdfsSnapshotReplicatorTest extends HdfsSnapshotReplicator {
@Test
public void replicationTest() throws Exception {
- Configuration sourceConf = ClusterHelper.getConfiguration(sourceCluster);
- this.setConf(sourceConf);
- Configuration targetConf = ClusterHelper.getConfiguration(targetCluster);
sourceStorageUrl = ClusterHelper.getStorageUrl(sourceCluster);
targetStorageUrl = ClusterHelper.getStorageUrl(targetCluster);
- DistributedFileSystem sourceFs = HadoopClientFactory.get().createDistributedProxiedFileSystem(sourceConf);
- DistributedFileSystem targetFs = HadoopClientFactory.get().createDistributedProxiedFileSystem(targetConf);
+ DistributedFileSystem sourceFs = HdfsSnapshotUtil.getSourceFileSystem(cmd,
+ new Configuration(getConf()));
+ DistributedFileSystem targetFs = HdfsSnapshotUtil.getTargetFileSystem(cmd,
+ new Configuration(getConf()));
// create dir1, create snapshot, invoke copy, check file in target, create snapshot on target
Path dir1 = new Path(sourceDir, "dir1");
http://git-wip-us.apache.org/repos/asf/falcon/blob/3ed804e9/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java b/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java
index 9e16fa4..f89def3 100644
--- a/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java
+++ b/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java
@@ -75,9 +75,8 @@ public final class ClusterHelper {
return conf;
}
- public static Configuration getConfiguration(String storageUrl, String executeEndPoint,
- String kerberosPrincipal) {
- Configuration conf = new Configuration();
+ public static Configuration getConfiguration(Configuration conf, String storageUrl,
+ String executeEndPoint, String kerberosPrincipal) {
conf.set(HadoopClientFactory.FS_DEFAULT_NAME_KEY, storageUrl);
conf.set(HadoopClientFactory.MR_JT_ADDRESS_KEY, executeEndPoint);
conf.set(HadoopClientFactory.YARN_RM_ADDRESS_KEY, executeEndPoint);